go_service/repositories/ImportRepository.go

package repositories

import (
	gm "db_service/gorm_models"
	"db_service/models"
	helper "db_service/pkg"
	"encoding/json"
	"errors"
	"fmt"
	"gorm.io/driver/mysql"
	"gorm.io/gorm"
	"log"
	"math"
	"os"
	"strconv"
	"sync"
	"time"
)

type Importer struct {
	mainCategories []gm.Category
	baza           *gorm.DB
	families       []gm.AttributeFamily
	sellers        map[string]gm.MarketplaceSeller
	AttributesMap  map[string]gm.Attribute
	Error          error
	ImportWGroup   sync.WaitGroup
	ColorOptions   map[string]gm.AttributeOption
	SexOptions     map[string]gm.AttributeOption
	ColorMutex     sync.Mutex
	SexMutex       sync.Mutex
}

func ImporterInstance() (instance *Importer, err error) {

	db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true})

	if err != nil {
		log.Println(err)
		return nil, err
	}

	instance = &Importer{baza: db}

	instance.ImportWGroup.Add(4)

	//load main categories to memory
	go func(db *gorm.DB) {
		defer instance.ImportWGroup.Done()
		instance.mainCategories, instance.Error = gm.GetMainCategories(db)
		log.Println(instance.mainCategories)
	}(db)

	//load families to memory
	go func() {
		defer instance.ImportWGroup.Done()
		instance.families, instance.Error = gm.GetFamilies(db)
	}()

	//load attributes to memory
	go func() {
		defer instance.ImportWGroup.Done()

		if attributes, err := gm.GetAttributes(db); err != nil {
			instance.Error = err
			return
		} else {
			instance.AttributesMap = make(map[string]gm.Attribute, len(attributes))

			for _, attribute := range attributes {
				instance.AttributesMap[attribute.Code] = attribute
			}
		}

		if colorOptions, err := gm.GetAttrOptions(db, instance.AttributesMap["color"].ID); err != nil {
			instance.Error = err
			return
		} else {
			instance.ColorOptions = make(map[string]gm.AttributeOption, len(colorOptions))

			for _, option := range colorOptions {
				instance.ColorOptions[option.AdminName] = option
			}
		}

		if sexOPtions, err := gm.GetAttrOptions(db, instance.AttributesMap["cinsiyet"].ID); err != nil {
			instance.Error = err
			return
		} else {
			instance.SexOptions = make(map[string]gm.AttributeOption, len(sexOPtions))

			for _, option := range sexOPtions {
				instance.SexOptions[option.AdminName] = option
			}
		}

	}()

	//load sellers to memory
	go func() {
		defer instance.ImportWGroup.Done()

		var vendors, err = gm.GetSellers(db)

		if err != nil {
			instance.Error = err
			return
		}
		//init sellers map
		instance.sellers = make(map[string]gm.MarketplaceSeller, len(vendors))

		for _, vendor := range vendors {
			instance.sellers[vendor.Url] = vendor
		}
	}()

	if instance.Error != nil {
		log.Println(instance.Error)
		return nil, instance.Error
	}

	return instance, nil
}

func (importer *Importer) Start() (instance *Importer) {

	log.Println("Start Product delete")
	deleteTime := time.Now()
	importer.Error = gm.DeleteProducts(importer.baza)
	deleteElapsed := time.Since(deleteTime)
	log.Printf("Delete products took %s", deleteElapsed)

	if importer.Error != nil {
		return importer
	}
	//init wait group to main categories length
	importer.ImportWGroup.Add(len(importer.mainCategories))
	//start gorutines for each main category
	for _, element := range importer.mainCategories {

		slug := element.Translations[0].Slug

		go importer.categoryRoutine("ty_db_" + slug)
	}

	return importer
}

func (importer *Importer) categoryRoutine(dbName string) {
	defer importer.ImportWGroup.Done()

	if dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName); dbExists {

		totalDocCount := getTotalDocumentCount(dbName)
		skip := 0
		limit := 200
		totalImport := 0
		for skip < totalDocCount {

			var response models.BagistoModelResponse

			url := fmt.Sprintf("%s%s/_all_docs?include_docs=true&limit=%v&skip=%v", os.Getenv("couch_db_source"), dbName, limit, skip)

			fmt.Println(url)

			skip += limit

			body, err := helper.SendRequest("GET", url, nil, "")

			if err != nil {
				fmt.Println(err.Error())
				continue
			}

			if err = json.Unmarshal(body, &response); err != nil {
				log.Println(err.Error())
				continue
			}

			//iterate 100 row products
			for _, element := range response.Rows {

				if err := importer.ImportProduct(element.Doc).Error; err != nil {
					log.Println(err)
				} else {
					totalImport++
				}
			}

			log.Printf("%s total imported documents count %d \n", dbName, totalImport)
		}
	} else {
		log.Println(dbName + "+doesnt exist")
	}

}

func getTotalDocumentCount(db string) int {

	var response models.DBDocCountResponse

	url := os.Getenv("couch_db_source") + db

	body, err := helper.SendRequest("GET", url, nil, "")

	if err != nil {
		log.Println(err.Error())
		return 0
	}

	err = json.Unmarshal(body, &response)
	if err != nil {
		log.Println(err.Error())
		return 0
	}

	return response.DocCount
}

func (importer *Importer) ImportProduct(product models.Product) (instance *Importer) {

	var linkedProducts []gm.Product
	var firstProduct *gm.Product
	if firstProduct, importer.Error = importer.importVariant(product); importer.Error != nil {
		return importer
	} else if product.ColorVariants != nil && len(*product.ColorVariants) > 0 {
		linkedProducts = append(linkedProducts, *firstProduct)
		for _, colorVariant := range *product.ColorVariants {

			if !colorVariant.IsSellable {
				continue
			}

			if variant, err := importer.importVariant(colorVariant); err == nil {
				linkedProducts = append(linkedProducts, *variant)
			}
		}
	}

	if len(linkedProducts) > 1 {

		var relation []gm.ProductRelation
		for index, variant := range linkedProducts {
			//spoint := "color" + strconv.Itoa(index)
			temp := make([]gm.Product, len(linkedProducts))

			copy(temp, linkedProducts)

			if index+1 <= len(temp) {
				temp = append(temp[:index], temp[index+1:]...)

				for _, item := range temp {
					relation = append(relation, gm.ProductRelation{ParentID: variant.ID, ChildID: item.ID})
				}
			}

		}

		if err := importer.baza.Create(&relation).Error; err != nil {
			log.Println(err)
		}

	}

	return importer
}

func (importer *Importer) importVariant(product models.Product) (*gm.Product, error) {
	productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))

	if categories, err := gm.GetCatKeywords(importer.baza, product.Categories); err != nil {
		return nil, err
	} else {
		productRepo.SetCategories(categories)
	}

	if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
		return nil, err
	} else {
		productRepo.Brand = brand
	}

	mainPorduct := productRepo.makeProduct(importer)

	//BEGIN TRANSACTION
	tx := importer.baza.Begin()

	if err := tx.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil {
		tx.Rollback()
		return nil, err
	}

	mainFlat := productRepo.makeProductFlat(mainPorduct.ID)

	if err := tx.Create(&mainFlat).Error; err != nil {
		tx.Rollback()
		return nil, err
	}

	if productRepo.HasSizeVariants() {
		var sizeVariants []gm.ProductFlat
		for index, variant := range *product.SizeVariants {
			if !variant.Sellable {
				continue
			}
			savePoint := "size" + strconv.Itoa(index)
			tx.SavePoint(savePoint)

			var sizeOPtion gm.AttributeOption

			if variant.AttributeName == "Beden" {
				sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue)
			} else {
				sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
			}

			sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
			variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku)

			variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)

			if err := tx.Omit("Categories.*").Create(&variantProduct).Error; err != nil {
				log.Println("Variant Product Create Error: " + err.Error())
				tx.RollbackTo(savePoint)
				continue
			}

			variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, mainFlat.ID, variantProduct.ID)

			if err := tx.Create(&variantFlat).Error; err != nil {
				log.Println("Variant Flat Create Error: " + err.Error())
				tx.RollbackTo(savePoint)
				continue
			}

			sizeVariants = append(sizeVariants, variantFlat)
		}

		if len(sizeVariants) == 0 {
			tx.Rollback()
			return nil, errors.New("siz variantlary yok bolsa main productam girayenok")
		} else {
			calcPrice(sizeVariants, &mainFlat)

			err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error

			mainFlat.Variants = sizeVariants

			if err != nil {
				tx.Rollback()
				return nil, err
			}
		}

	}

	sProduct := importer.createSellerProduct(&mainFlat, product.Vendor)

	if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil {
		tx.Rollback()
		return nil, errSProduct
	}

	if err := tx.Commit().Error; err != nil {
		return nil, err
	}

	return &mainPorduct, nil
}

func (importer *Importer) createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct {
	sellerID := importer.sellers[sellerURL].ID

	if sellerID == 0 {
		sellerID = 1
	}

	sellerProduct := gm.MarketplaceProduct{
		MarketplaceSellerID: sellerID,
		IsApproved:          true,
		Condition:           "new",
		Description:         "scraped",
		IsOwner:             true,
		ProductID:           flat.ProductID,
	}

	for _, variant := range flat.Variants {
		sellerProduct.Variants = append(sellerProduct.Variants, gm.MarketplaceProduct{
			ProductID:           variant.ProductID,
			IsOwner:             true,
			IsApproved:          true,
			MarketplaceSellerID: sellerID,
			Condition:           "new",
		})
	}

	return sellerProduct

}

func calcPrice(variants []gm.ProductFlat, flat *gm.ProductFlat) {
	for _, variant := range variants {

		if !variant.Status {
			continue
		}

		if flat.MinPrice == 0 || flat.MinPrice > variant.MinPrice {
			flat.MinPrice = variant.MinPrice
		}

		flat.MaxPrice = math.Max(flat.MaxPrice, variant.MaxPrice)
	}
}

func (importer *Importer) GetColorOption(optionName string) gm.AttributeOption {
	if optionName == "" {
		return gm.AttributeOption{}
	}

	importer.ColorMutex.Lock()
	var option gm.AttributeOption
	var ok bool

	if option, ok = importer.ColorOptions[optionName]; !ok {
		option := gm.GetAttributeOption(importer.baza, importer.AttributesMap["color"].ID, optionName)

		importer.ColorOptions[optionName] = option
	}
	importer.ColorMutex.Unlock()
	return option
}

func (importer *Importer) GetSexOption(optionName string) gm.AttributeOption {
	if optionName == "" {
		return gm.AttributeOption{}
	}
	importer.SexMutex.Lock()
	var option gm.AttributeOption
	var ok bool

	if option, ok = importer.ColorOptions[optionName]; !ok {
		option = gm.GetAttributeOption(importer.baza, importer.AttributesMap["cinsiyet"].ID, optionName)
		importer.SexOptions[optionName] = option
	}
	importer.SexMutex.Unlock()
	return option
}

func (importer *Importer) updateVariant(product models.Product) (*gm.Product, error) {

	var flat gm.ProductFlat
	err := importer.baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error
	if err != nil {
		return nil, err
	}

	if flat.Product.Type == "configurable" {

		productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
		if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
			return nil, err
		} else {
			productRepo.Brand = brand
		}

		for _, variant := range *product.SizeVariants {

			found := false
			for _, flatVariant := range flat.Variants {

				if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel {

					if !variant.Sellable {
						importer.baza.Model(&flatVariant).Update("status", false)
					} else {

						importer.updatePrice(variant.Price, &flatVariant)

						if !flatVariant.Status {

							importer.baza.Model(&flatVariant).Update("status", true)
						}
					}
					found = true
					break
				}
			}

			if variant.Sellable && !found {
				// insert variant
				var sizeOPtion gm.AttributeOption

				if variant.AttributeName == "Beden" {
					sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["size"].ID, variant.AttributeValue)
				} else {
					sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
				}

				sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
				variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku)

				variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)

				if err := importer.baza.Omit("Categories.*").Create(&variantProduct).Error; err != nil {
					log.Println("Variant Product Create Error: " + err.Error())
				}

				variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID)

				if err := importer.baza.Create(&variantFlat).Error; err != nil {
					log.Println("Variant Flat Create Error: " + err.Error())

				}
				flat.Variants = append(flat.Variants, variantFlat)
			}
		}

		calcPrice(flat.Variants, &flat)
		importer.baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat)

	} else { //simple

		importer.updatePrice(product.Price, &flat)

	}
	return &flat.Product, nil
}

func (importer *Importer) updatePrice(price models.Price, flat *gm.ProductFlat) {
	if price.OriginalPrice.Value > price.DiscountedPrice.Value {

		importer.baza.Model(&flat).Updates(map[string]interface{}{
			"price":         price.OriginalPrice.Value,
			"special_price": price.DiscountedPrice.Value,
			"min_price":     price.DiscountedPrice.Value,
			"max_price":     price.OriginalPrice.Value,
		})

		importer.baza.Model(&gm.ProductAttributeValue{}).
			Where("attribute_id = 11 and product_id = ?", flat.ProductID).
			Update("float_value", price.OriginalPrice.Value)

		importer.baza.Model(&gm.ProductAttributeValue{}).
			Where("attribute_id = 13 and product_id = ?", flat.ProductID).
			Update("float_value", price.DiscountedPrice.Value)
	} else {

		importer.baza.Model(&flat).Updates(map[string]interface{}{
			"price":         price.DiscountedPrice.Value,
			"special_price": nil,
			"min_price":     price.DiscountedPrice.Value,
			"max_price":     price.DiscountedPrice.Value,
		})

		importer.baza.Model(&gm.ProductAttributeValue{}).
			Where("attribute_id = 11 and product_id = ?", flat.ProductID).
			Update("float_value", price.DiscountedPrice.Value)

		importer.baza.Where("attribute_id = 13 and product_id = ?", flat.ProductID).
			Delete(&gm.ProductAttributeValue{})
	}
}

func (importer *Importer) UpdateOrCreate(product models.Product) (instance *Importer) {

	firstProduct, err := importer.importVariant(product)
	var newProducts []gm.Product
	if err != nil {
		helper.Error(err)

		firstProduct, importer.Error = importer.updateVariant(product)

		if importer.Error != nil {
			return importer
		}
	} else if &firstProduct != nil {
		newProducts = append(newProducts, *firstProduct)
	}

	if product.ColorVariants != nil && len(*product.ColorVariants) > 0 {
		linkedProducts := []gm.Product{*firstProduct}

		for _, colorVariant := range *product.ColorVariants {

			var (
				variant *gm.Product
				err     error
			)
			if !colorVariant.IsSellable {

				if err = importer.baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil {
					log.Println(err)
				}
				continue

			} else {

				if variant, err = importer.importVariant(colorVariant); err != nil {

					if variant, importer.Error = importer.updateVariant(colorVariant); importer.Error != nil {
						return importer
					}

					linkedProducts = append(linkedProducts, *variant)

				} else {
					newProducts = append(newProducts, *variant)
				}
			}

		}

		if len(newProducts) > 0 {
			// relation

			var relation []gm.ProductRelation
			for _, linkedProduct := range linkedProducts {

				for _, newProd := range newProducts {
					relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID})
					relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID})
				}
			}

			if err := importer.baza.Create(&relation).Error; err != nil {
				log.Println(err)
			}
		}

	}
	return importer
}