go_service/repositories/ImportRepository.go

package repositories

import (
	gm "db_service/gorm_models"
	"db_service/models"
	helper "db_service/pkg"
	"encoding/json"
	"errors"
	"fmt"
	"gorm.io/driver/mysql"
	"gorm.io/gorm"
	"log"
	"math"
	"os"
	"sync"
)

type Importer struct {
	mainCategories []gm.Category
	baza           *gorm.DB
	families       []gm.AttributeFamily
	sellers        map[string]gm.MarketplaceSeller
	AttributesMap  map[string]gm.Attribute
	Error          error
	ImportWGroup   sync.WaitGroup
	ColorOptions   map[string]gm.AttributeOption
	SexOptions     map[string]gm.AttributeOption
}

func ImporterInstance() (instance *Importer, err error) {

	db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true})

	if err != nil {
		log.Println(err)
		return nil, err
	}

	instance = &Importer{baza: db}

	instance.ImportWGroup.Add(4)

	//load main categories to memory
	go func(db *gorm.DB) {
		defer instance.ImportWGroup.Done()
		instance.mainCategories, instance.Error = gm.GetMainCategories(db)
	}(db)

	//load families to memory
	go func() {
		defer instance.ImportWGroup.Done()
		instance.families, instance.Error = gm.GetFamilies(db)
	}()

	//load attributes to memory
	go func() {
		defer instance.ImportWGroup.Done()

		if attributes, err := gm.GetAttributes(db); err != nil {
			instance.Error = err
			return
		} else {
			instance.AttributesMap = make(map[string]gm.Attribute, len(attributes))

			for _, attribute := range attributes {
				instance.AttributesMap[attribute.Code] = attribute
			}
		}

		if colorOptions, err := gm.GetAttrOptions(db, instance.AttributesMap["color"].ID); err != nil {
			instance.Error = err
			return
		} else {
			instance.ColorOptions = make(map[string]gm.AttributeOption, len(colorOptions))

			for _, option := range colorOptions {
				instance.ColorOptions[option.AdminName] = option
			}
		}

		if sexOPtions, err := gm.GetAttrOptions(db, instance.AttributesMap["cinsiyet"].ID); err != nil {
			instance.Error = err
			return
		} else {
			instance.SexOptions = make(map[string]gm.AttributeOption, len(sexOPtions))

			for _, option := range sexOPtions {
				instance.SexOptions[option.AdminName] = option
			}
		}

	}()

	//load sellers to memory
	go func() {
		defer instance.ImportWGroup.Done()

		var vendors, err = gm.GetSellers(db)

		if err != nil {
			instance.Error = err
			return
		}
		//init sellers map
		instance.sellers = make(map[string]gm.MarketplaceSeller, len(vendors))

		for _, vendor := range vendors {
			instance.sellers[vendor.Url] = vendor
		}
	}()

	if instance.Error != nil {
		log.Println(instance.Error)
		return nil, instance.Error
	}

	return instance, nil
}

func (importer *Importer) Start() (instance *Importer) {
	//init wait group to main categories length
	importer.ImportWGroup.Add(len(importer.mainCategories))
	//start gorutines for each main category
	for _, element := range importer.mainCategories {

		slug := element.Translations[0].Slug

		go importer.categoryRoutine("ty_db_" + slug)
	}

	return importer
}

func (importer *Importer) categoryRoutine(dbName string) {
	defer importer.ImportWGroup.Done()

	if dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName); dbExists {

		totalDocCount := getTotalDocumentCount(dbName)
		skip := 0
		limit := 200
		totalImport := 0
		for skip < totalDocCount {

			var response models.BagistoModelResponse

			url := fmt.Sprintf("%s%s/_all_docs?include_docs=true&limit=%v&skip=%v", os.Getenv("couch_db_source"), dbName, limit, skip)

			fmt.Println(url)

			skip += limit

			body, err := helper.SendRequest("GET", url, nil, "")

			if err != nil {
				fmt.Println(err.Error())
				continue
			}

			if err = json.Unmarshal(body, &response); err != nil {
				log.Println(err.Error())
				continue
			}

			//iterate 100 row products
			for _, element := range response.Rows {

				if err := importer.ImportProduct(element.Doc).Error; err != nil {
					log.Println(err)
				} else {
					totalImport++
				}
			}

			log.Printf("%s total imported documents count %d \n", dbName, totalImport)
		}
	} else {
		log.Println(dbName + "+doesnt exist")
	}

}

func getTotalDocumentCount(db string) int {

	var response models.DBDocCountResponse

	url := os.Getenv("couch_db_source") + db

	body, err := helper.SendRequest("GET", url, nil, "")

	if err != nil {
		log.Println(err.Error())
		return 0
	}

	err = json.Unmarshal(body, &response)
	if err != nil {
		log.Println(err.Error())
		return 0
	}

	return response.DocCount
}

func (importer *Importer) ImportProduct(product models.Product) (instance *Importer) {

	var linkedProducts []gm.Product
	if firstProduct, err := importer.importVariant(product); err == nil {
		linkedProducts = append(linkedProducts, *firstProduct)
	}

	if product.ColorVariants != nil && len(*product.ColorVariants) > 0 {
		for _, colorVariant := range *product.ColorVariants {

			if variant, err := importer.importVariant(colorVariant); err == nil {
				linkedProducts = append(linkedProducts, *variant)
			}
		}
	}

	if len(linkedProducts) > 0 {
		//todo link products
	}

	return importer
}

func (importer *Importer) importVariant(product models.Product) (*gm.Product, error) {
	productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))

	if categories, err := gm.GetCatKeywords(importer.baza, product.Categories); err != nil {
		return nil, err
	} else {
		productRepo.SetCategories(categories)
	}

	if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
		return nil, err
	} else {
		productRepo.Brand = brand
	}

	mainPorduct := productRepo.makeProduct(importer)

	//BEGIN TRANSACTION
	tx := *importer.baza.Begin()

	importer.Error = tx.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error

	if err := tx.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil {
		tx.Rollback()
		return nil, err
	}

	mainFlat := productRepo.makeProductFlat(mainPorduct.ID)

	if err := tx.Create(&mainFlat).Error; err != nil {
		tx.Rollback()
		return nil, err
	}

	if productRepo.HasSizeVariants() {
		var sizeVariants []gm.ProductFlat
		for _, variant := range *product.SizeVariants {
			tx.SavePoint(variant.ListingID)
			sizeOPtion := gm.GetAttributeOption(&tx, importer.AttributesMap["size"].ID, variant.AttributeValue)

			variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID)

			variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)

			if err := tx.Omit("Categories.*").Create(&variantProduct).Error; err != nil {
				log.Println("Variant Product Create Error: " + err.Error())
				tx.RollbackTo(variant.ListingID)
				continue
			}

			variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, mainFlat.ID)

			if err := tx.Create(&variantFlat).Error; err != nil {
				log.Println("Variant Flat Create Error: " + err.Error())
				tx.RollbackTo(variant.ListingID)
				continue
			}

			sizeVariants = append(sizeVariants, variantFlat)
		}

		if len(sizeVariants) > 0 {
			tx.Rollback()
			return nil, errors.New("siz variantlary yok bolsa main productam girayenok")
		} else {
			calcPrice(sizeVariants, &mainFlat)

			err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error
			if err != nil {
				tx.Rollback()
				return nil, err
			}
		}

	}

	if err := tx.Commit().Error; err != nil {
		return nil, err
	}

	return &mainPorduct, nil
}

func calcPrice(variants []gm.ProductFlat, flat *gm.ProductFlat) {
	for _, variant := range variants {

		if flat.MinPrice == 0 || flat.MinPrice > variant.MinPrice {
			flat.MinPrice = variant.MinPrice
		}

		flat.MaxPrice = math.Max(flat.MaxPrice, variant.MaxPrice)
	}
}

func (importer *Importer) GetColorOption(optionName string) gm.AttributeOption {
	if optionName == "" {
		return gm.AttributeOption{}
	}

	if option, ok := importer.ColorOptions[optionName]; ok {
		return option
	} else {
		option := gm.GetAttributeOption(importer.baza, importer.AttributesMap["color"].ID, optionName)

		importer.ColorOptions[optionName] = option
		return option
	}
}

func (importer *Importer) GetSexOption(optionName string) gm.AttributeOption {
	if optionName == "" {
		return gm.AttributeOption{}
	}

	if option, ok := importer.ColorOptions[optionName]; ok {
		return option
	} else {
		option := gm.GetAttributeOption(importer.baza, importer.AttributesMap["cinsiyet"].ID, optionName)

		importer.SexOptions[optionName] = option
		return option
	}
}