go_service/repositories/ImportRepository.go

package repositories

import (
	gm "db_service/gorm_models"
	"db_service/models"
	helper "db_service/pkg"
	"encoding/json"
	"fmt"
	"gorm.io/driver/mysql"
	"gorm.io/gorm"
	"log"
	"os"
	"sync"
)

type Importer struct {
	mainCategories []gm.Category
	baza           *gorm.DB
	families       []gm.AttributeFamily
	sellers        map[string]gm.MarketplaceSeller
	AttributesMap  map[string]gm.Attribute
	Error          error
	ImportWGroup   sync.WaitGroup
	ColorOptions   map[string]gm.AttributeOption
	SexOptions     map[string]gm.AttributeOption
}

func ImporterInstance() (instance *Importer, err error) {

	db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true})

	if err != nil {
		log.Println(err)
		return nil, err
	}

	instance = &Importer{baza: db}

	instance.ImportWGroup.Add(4)

	//load main categories to memory
	go func(db *gorm.DB) {
		defer instance.ImportWGroup.Done()
		instance.mainCategories, instance.Error = gm.GetMainCategories(db)
	}(db)

	//load families to memory
	go func() {
		defer instance.ImportWGroup.Done()
		instance.families, instance.Error = gm.GetFamilies(db)
	}()

	//load attributes to memory
	go func() {
		defer instance.ImportWGroup.Done()

		if attributes, err := gm.GetAttributes(db); err != nil {
			instance.Error = err
			return
		} else {
			instance.AttributesMap = make(map[string]gm.Attribute, len(attributes))

			for _, attribute := range attributes {
				instance.AttributesMap[attribute.Code] = attribute
			}
		}

		if colorOptions, err := gm.GetAttrOptions(db, instance.AttributesMap["color"].ID); err != nil {
			instance.Error = err
			return
		} else {
			instance.ColorOptions = make(map[string]gm.AttributeOption, len(colorOptions))

			for _, option := range colorOptions {
				instance.ColorOptions[option.AdminName] = option
			}
		}

		if sexOPtions, err := gm.GetAttrOptions(db, instance.AttributesMap["cinsiyet"].ID); err != nil {
			instance.Error = err
			return
		} else {
			instance.SexOptions = make(map[string]gm.AttributeOption, len(sexOPtions))

			for _, option := range sexOPtions {
				instance.SexOptions[option.AdminName] = option
			}
		}

	}()

	//load sellers to memory
	go func() {
		defer instance.ImportWGroup.Done()

		var vendors, err = gm.GetSellers(db)

		if err != nil {
			instance.Error = err
			return
		}
		//init sellers map
		instance.sellers = make(map[string]gm.MarketplaceSeller, len(vendors))

		for _, vendor := range vendors {
			instance.sellers[vendor.Url] = vendor
		}
	}()

	if instance.Error != nil {
		log.Println(instance.Error)
		return nil, instance.Error
	}

	return instance, nil
}

func (importer *Importer) Start() (instance *Importer) {
	//init wait group to main categories length
	importer.ImportWGroup.Add(len(importer.mainCategories))
	//start gorutines for each main category
	for _, element := range importer.mainCategories {

		slug := element.Translations[0].Slug

		go importer.categoryRoutine("ty_db_" + slug)
	}

	return importer
}

func (importer *Importer) categoryRoutine(dbName string) {
	defer importer.ImportWGroup.Done()

	if dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName); dbExists {

		totalDocCount := getTotalDocumentCount(dbName)
		skip := 0
		limit := 200
		totalImport := 0
		for skip < totalDocCount {

			var response models.BagistoModelResponse

			url := fmt.Sprintf("%s%s/_all_docs?include_docs=true&limit=%v&skip=%v", os.Getenv("couch_db_source"), dbName, limit, skip)

			fmt.Println(url)

			skip += limit

			body, err := helper.SendRequest("GET", url, nil, "")

			if err != nil {
				fmt.Println(err.Error())
				continue
			}

			if err = json.Unmarshal(body, &response); err != nil {
				log.Println(err.Error())
				continue
			}

			//iterate 100 row products
			for _, element := range response.Rows {

				if err := importer.ImportProduct(element.Doc).Error; err != nil {
					log.Println(err)
				} else {
					totalImport++
				}
			}

			log.Printf("%s total imported documents count %d \n", dbName, totalImport)
		}
	} else {
		log.Println(dbName + "+doesnt exist")
	}

}

func getTotalDocumentCount(db string) int {

	var response models.DBDocCountResponse

	url := os.Getenv("couch_db_source") + db

	body, err := helper.SendRequest("GET", url, nil, "")

	if err != nil {
		log.Println(err.Error())
		return 0
	}

	err = json.Unmarshal(body, &response)
	if err != nil {
		log.Println(err.Error())
		return 0
	}

	return response.DocCount
}

func (importer *Importer) ImportProduct(product models.Product) (instance *Importer) {

	productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
	var categories []gm.Category

	if categories, importer.Error = gm.GetCatKeywords(importer.baza, product.Categories); importer.Error != nil {
		log.Println("ERR0 Categories" + importer.Error.Error())
		return importer
	}

	productRepo.SetCategories(categories)

	if productRepo.Brand, importer.Error = gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); importer.Error != nil {
		log.Println("ERR BRAND" + importer.Error.Error())
		return importer
	}

	//BEGIN TRANSACTION
	//tx := *importer.baza.Begin()

	return importer
}

func (importer *Importer) GetColorOption(optionName string) gm.AttributeOption {
	if optionName == "" {
		return gm.AttributeOption{}
	}

	if option, ok := importer.ColorOptions[optionName]; ok {
		return option
	} else {
		option := gm.GetAttributeOption(importer.baza, importer.AttributesMap["color"].ID, optionName)

		importer.ColorOptions[optionName] = option
		return option
	}
}

func (importer *Importer) GetSexOption(optionName string) gm.AttributeOption {
	if optionName == "" {
		return gm.AttributeOption{}
	}

	if option, ok := importer.ColorOptions[optionName]; ok {
		return option
	} else {
		option := gm.GetAttributeOption(importer.baza, importer.AttributesMap["cinsiyet"].ID, optionName)

		importer.SexOptions[optionName] = option
		return option
	}
}