package repositories import ( "encoding/json" "errors" "fmt" "gorm.io/gorm" "log" "math" gm "sarga_updater/bagisto_models" helper "sarga_updater/helpers" models "sarga_updater/trendyol_models" "strconv" "sync" ) type Importer struct { mainCategories []gm.Category Baza *gorm.DB families []gm.AttributeFamily sellers map[string]gm.MarketplaceSeller //wishlist map[string]gm.Product AttributesMap map[string]gm.Attribute Error error ImportWGroup sync.WaitGroup ColorOptions map[string]gm.AttributeOption SexOptions map[string]gm.AttributeOption ColorMutex sync.Mutex SexMutex sync.Mutex } type GormErr struct { Number int `json:"Number"` Message string `json:"Message"` } func ParseImporterInstance(db *gorm.DB) (instance *Importer, err error) { if err != nil { log.Println(err) return nil, err } instance = &Importer{Baza: db} instance.ImportWGroup.Add(3) //load families to memory go func() { defer instance.ImportWGroup.Done() instance.families, instance.Error = gm.GetFamilies(db) }() //load attributes to memory go func() { defer instance.ImportWGroup.Done() if attributes, err := gm.GetAttributes(db); err != nil { instance.Error = err return } else { instance.AttributesMap = make(map[string]gm.Attribute, len(attributes)) for _, attribute := range attributes { instance.AttributesMap[attribute.Code] = attribute } } if colorOptions, err := gm.GetAttrOptions(db, instance.AttributesMap["color"].ID); err != nil { instance.Error = err return } else { instance.ColorOptions = make(map[string]gm.AttributeOption, len(colorOptions)) for _, option := range colorOptions { instance.ColorOptions[option.AdminName] = option } } if sexOPtions, err := gm.GetAttrOptions(db, instance.AttributesMap["cinsiyet"].ID); err != nil { instance.Error = err return } else { instance.SexOptions = make(map[string]gm.AttributeOption, len(sexOPtions)) for _, option := range sexOPtions { instance.SexOptions[option.AdminName] = option } } }() //load sellers to memory go func() { defer instance.ImportWGroup.Done() var vendors, err = gm.GetSellers(db) if err != nil { instance.Error = err return } //init sellers map instance.sellers = make(map[string]gm.MarketplaceSeller, len(vendors)) for _, vendor := range vendors { instance.sellers[vendor.Url] = vendor } }() if instance.Error != nil { log.Println(instance.Error) return nil, instance.Error } return instance, nil } func (importer *Importer) ImportProduct(product models.Product) (instance *Importer) { var linkedProducts []gm.Product var firstProduct *gm.Product if firstProduct, importer.Error = importer.importVariant(product); importer.Error != nil { return importer } else if product.ColorVariants != nil && len(*product.ColorVariants) > 0 { linkedProducts = append(linkedProducts, *firstProduct) for _, colorVariant := range *product.ColorVariants { if !colorVariant.IsSellable { continue } if variant, err := importer.importVariant(colorVariant); err == nil { linkedProducts = append(linkedProducts, *variant) } } } if len(linkedProducts) > 1 { var relation []gm.ProductRelation for index, variant := range linkedProducts { //spoint := "color" + strconv.Itoa(index) temp := make([]gm.Product, len(linkedProducts)) copy(temp, linkedProducts) if index+1 <= len(temp) { temp = append(temp[:index], temp[index+1:]...) for _, item := range temp { relation = append(relation, gm.ProductRelation{ParentID: variant.ID, ChildID: item.ID}) } } } if err := importer.Baza.Create(&relation).Error; err != nil { log.Println(err) } } return importer } func (importer *Importer) importVariant(product models.Product) (*gm.Product, error) { // check if wishlisted then update if. //if _, ok := importer.wishlist[product.ProductNumber]; ok { // delete(importer.wishlist,product.ProductNumber) // return importer.updateVariant(product) //} productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet)) if categories, err := gm.GetCatKeywords(importer.Baza, product.Categories); err != nil { return nil, err } else { productRepo.SetCategories(categories) } if brand, err := gm.FindOrCreateBrand(importer.Baza, product.Brand, productRepo.Categories); err != nil { return nil, err } else { productRepo.Brand = brand } mainPorduct := productRepo.makeProduct(importer) //BEGIN TRANSACTION tx := importer.Baza.Begin() if err := tx.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil { //todo update categories byteErr, _ := json.Marshal(err) var newError GormErr if err1 := json.Unmarshal((byteErr), &newError); err1 != nil { tx.Rollback() log.Println(err1, "err2") return nil, err1 } if newError.Number == 1062 { var barProduct gm.Product if err2 := tx.First(&barProduct, "sku = ?", mainPorduct.Sku).Error; err2 != nil { tx.Rollback() log.Println(err2, "err3") return nil, err2 } if err3 := tx.Model(&barProduct).Association("Categories").Append(mainPorduct.Categories); err3 != nil { tx.Rollback() log.Println(err3, "err4") return nil, err3 } if err4 := tx.Commit().Error; err4 == nil { return importer.updateVariant(product) } } else { tx.Rollback() log.Println(err, "er1") return nil, err } } mainFlat := productRepo.makeProductFlat(mainPorduct.ID) if err := tx.Create(&mainFlat).Error; err != nil { tx.Rollback() log.Println(err, "er5") return nil, err } if productRepo.HasSizeVariants() { var sizeVariants []gm.ProductFlat for index, variant := range *product.SizeVariants { if !variant.Sellable { continue } savePoint := "size" + strconv.Itoa(index) tx.SavePoint(savePoint) var sizeOPtion gm.AttributeOption if variant.AttributeName == "Beden" { sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue) } else { sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue) } sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber) variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku) variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID) if err := tx.Omit("Categories.*").Create(&variantProduct).Error; err != nil { log.Println("Variant Product Create Error: " + err.Error()) tx.RollbackTo(savePoint) continue } variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, mainFlat.ID, variantProduct.ID) if err := tx.Create(&variantFlat).Error; err != nil { log.Println("Variant Flat Create Error: " + err.Error()) tx.RollbackTo(savePoint) continue } sizeVariants = append(sizeVariants, variantFlat) } if len(sizeVariants) == 0 { tx.Rollback() return nil, errors.New("size variantlary yok bolsa main productam girayenok") } else { calcPrice(sizeVariants, &mainFlat) err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error mainFlat.Variants = sizeVariants if err != nil { tx.Rollback() log.Println(err, "er6") return nil, err } } } sProduct := importer.createSellerProduct(&mainFlat, product.Vendor) if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil { tx.Rollback() return nil, errSProduct } if err := tx.Commit().Error; err != nil { return nil, err } return &mainPorduct, nil } func (importer *Importer) createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct { sellerID := importer.sellers[sellerURL].ID if sellerID == 0 { sellerID = 1 } sellerProduct := gm.MarketplaceProduct{ MarketplaceSellerID: sellerID, IsApproved: true, Condition: "new", Description: "scraped", IsOwner: true, ProductID: flat.ProductID, } for _, variant := range flat.Variants { sellerProduct.Variants = append(sellerProduct.Variants, gm.MarketplaceProduct{ ProductID: variant.ProductID, IsOwner: true, IsApproved: true, MarketplaceSellerID: sellerID, Condition: "new", }) } return sellerProduct } func calcPrice(variants []gm.ProductFlat, flat *gm.ProductFlat) { for _, variant := range variants { if !variant.Status { continue } if flat.MinPrice == 0 || flat.MinPrice > variant.MinPrice { flat.MinPrice = variant.MinPrice } flat.MaxPrice = math.Max(flat.MaxPrice, variant.MaxPrice) } } func (importer *Importer) GetColorOption(optionName string) gm.AttributeOption { if optionName == "" { return gm.AttributeOption{} } importer.ColorMutex.Lock() var option gm.AttributeOption var ok bool if option, ok = importer.ColorOptions[optionName]; !ok { option := gm.GetAttributeOption(importer.Baza, importer.AttributesMap["color"].ID, optionName) importer.ColorOptions[optionName] = option } importer.ColorMutex.Unlock() return option } func (importer *Importer) GetSexOption(optionName string) gm.AttributeOption { if optionName == "" { return gm.AttributeOption{} } importer.SexMutex.Lock() var option gm.AttributeOption var ok bool if option, ok = importer.ColorOptions[optionName]; !ok { option = gm.GetAttributeOption(importer.Baza, importer.AttributesMap["cinsiyet"].ID, optionName) importer.SexOptions[optionName] = option } importer.SexMutex.Unlock() return option } func (importer *Importer) updateVariant(product models.Product) (*gm.Product, error) { var flat gm.ProductFlat err := importer.Baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return importer.importVariant(product) } //todo not found bolsa create etmeli return nil, err } if flat.Product.Type == "configurable" { productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet)) if brand, err := gm.FindOrCreateBrand(importer.Baza, product.Brand, productRepo.Categories); err != nil { return nil, err } else { productRepo.Brand = brand } for _, variant := range *product.SizeVariants { found := false for _, flatVariant := range flat.Variants { if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel { if !variant.Sellable { importer.Baza.Model(&flatVariant).Update("status", false) } else { importer.updatePrice(variant.Price, &flatVariant) if !flatVariant.Status { importer.Baza.Model(&flatVariant).Update("status", true) } } found = true break } } if variant.Sellable && !found { // insert variant var sizeOPtion gm.AttributeOption if variant.AttributeName == "Beden" { sizeOPtion = gm.GetAttributeOption(importer.Baza, importer.AttributesMap["size"].ID, variant.AttributeValue) } else { sizeOPtion = gm.GetAttributeOption(importer.Baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue) } sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber) variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku) variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID) if err := importer.Baza.Omit("Categories.*").Create(&variantProduct).Error; err != nil { log.Println("Variant Product Create Error: " + err.Error()) } else { variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID) if err := importer.Baza.Create(&variantFlat).Error; err != nil { log.Println("Variant Flat Create Error: " + err.Error()) } flat.Variants = append(flat.Variants, variantFlat) } } } calcPrice(flat.Variants, &flat) importer.Baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat) } else { //simple importer.updatePrice(product.Price, &flat) } return &flat.Product, nil } func (importer *Importer) updatePrice(price models.Price, flat *gm.ProductFlat) { if price.OriginalPrice.Value > price.DiscountedPrice.Value { importer.Baza.Model(&flat).Updates(map[string]interface{}{ "price": price.OriginalPrice.Value, "special_price": price.DiscountedPrice.Value, "min_price": price.DiscountedPrice.Value, "max_price": price.OriginalPrice.Value, }) importer.Baza.Model(&gm.ProductAttributeValue{}). Where("attribute_id = 11 and product_id = ?", flat.ProductID). Update("float_value", price.OriginalPrice.Value) importer.Baza.Model(&gm.ProductAttributeValue{}). Where("attribute_id = 13 and product_id = ?", flat.ProductID). Update("float_value", price.DiscountedPrice.Value) } else { importer.Baza.Model(&flat).Updates(map[string]interface{}{ "price": price.DiscountedPrice.Value, "special_price": nil, "min_price": price.DiscountedPrice.Value, "max_price": price.DiscountedPrice.Value, }) importer.Baza.Model(&gm.ProductAttributeValue{}). Where("attribute_id = 11 and product_id = ?", flat.ProductID). Update("float_value", price.DiscountedPrice.Value) importer.Baza.Where("attribute_id = 13 and product_id = ?", flat.ProductID). Delete(&gm.ProductAttributeValue{}) } } func (importer *Importer) importParsedVariant(product models.Product) (gm.Product, error) { //todo search if exists productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet)) productRepo.ImageType = "lcw" if brand, err := gm.FindOrCreateBrand(importer.Baza, product.Brand, productRepo.Categories); err != nil { return gm.Product{}, err } else { productRepo.Brand = brand } mainPorduct := productRepo.makeProduct(importer) //BEGIN TRANSACTION tx := importer.Baza.Begin() if err := tx.Omit("Categories", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil { tx.Rollback() return gm.Product{}, err } mainFlat := productRepo.makeProductFlat(mainPorduct.ID) if err := tx.Create(&mainFlat).Error; err != nil { tx.Rollback() return gm.Product{}, err } if productRepo.HasSizeVariants() { var sizeVariants []gm.ProductFlat for index, variant := range *product.SizeVariants { if !variant.Sellable { continue } savePoint := "size" + strconv.Itoa(index) tx.SavePoint(savePoint) var sizeOPtion gm.AttributeOption if variant.AttributeName == "Beden" { sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue) } else { sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue) } sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber) log.Println(sku) log.Println(variant) variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku) variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID) if err := tx.Omit("Categories.*").Create(&variantProduct).Error; err != nil { log.Println("Variant Product Create Error: " + err.Error()) tx.RollbackTo(savePoint) continue } variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, mainFlat.ID, variantProduct.ID) if err := tx.Create(&variantFlat).Error; err != nil { log.Println("Variant Flat Create Error: " + err.Error()) tx.RollbackTo(savePoint) continue } sizeVariants = append(sizeVariants, variantFlat) } if len(sizeVariants) == 0 { tx.Rollback() return gm.Product{}, errors.New("siz variantlary yok bolsa main productam girayenok") } else { calcPrice(sizeVariants, &mainFlat) err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error mainFlat.Variants = sizeVariants if err != nil { tx.Rollback() return gm.Product{}, err } } } log.Println(product.Vendor) sProduct := importer.createSellerProduct(&mainFlat, product.Vendor) if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil { tx.Rollback() return gm.Product{}, errSProduct } if err := tx.Commit().Error; err != nil { return gm.Product{}, err } return mainPorduct, nil } func (importer *Importer) updateParsedVariant(product models.Product) (gm.Product, error) { var flat gm.ProductFlat err := importer.Baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return importer.importParsedVariant(product) } //todo not found bolsa create etmeli return gm.Product{}, err } if flat.Product.Type == "configurable" { productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet)) productRepo.ImageType = "lcw" if brand, err := gm.FindOrCreateBrand(importer.Baza, product.Brand, productRepo.Categories); err != nil { return gm.Product{}, err } else { productRepo.Brand = brand } for _, variant := range *product.SizeVariants { found := false for _, flatVariant := range flat.Variants { if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel { if !variant.Sellable { importer.Baza.Model(&flatVariant).Update("status", false) } else { importer.updatePrice(variant.Price, &flatVariant) if !flatVariant.Status { importer.Baza.Model(&flatVariant).Update("status", true) } } found = true break } } if variant.Sellable && !found { // insert variant var sizeOPtion gm.AttributeOption if variant.AttributeName == "Beden" { sizeOPtion = gm.GetAttributeOption(importer.Baza, importer.AttributesMap["size"].ID, variant.AttributeValue) } else { sizeOPtion = gm.GetAttributeOption(importer.Baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue) } log.Println(variant) sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber) log.Println(sku) variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku) variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID) if err := importer.Baza.Omit("Categories").Create(&variantProduct).Error; err != nil { log.Println("Variant Product Create Error: " + err.Error()) } variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID) if err := importer.Baza.Create(&variantFlat).Error; err != nil { log.Println("Variant Flat Create Error: " + err.Error()) } flat.Variants = append(flat.Variants, variantFlat) } } calcPrice(flat.Variants, &flat) importer.Baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat) } else { //simple importer.updatePrice(product.Price, &flat) } return flat.Product, nil } func (importer *Importer) UpdateOrCreateLCW(product models.Product) (instance *Importer) { var firstProduct gm.Product result := importer.Baza.First(&firstProduct, "sku=?", product.ProductNumber) if errors.Is(result.Error, gorm.ErrRecordNotFound) { firstProduct, importer.Error = importer.importParsedVariant(product) } else { firstProduct, importer.Error = importer.updateParsedVariant(product) } var newProducts []gm.Product if importer.Error != nil { return importer } else if &firstProduct != nil { newProducts = append(newProducts, firstProduct) } if product.ColorVariants != nil && len(*product.ColorVariants) > 0 { linkedProducts := []gm.Product{firstProduct} for _, colorVariant := range *product.ColorVariants { var ( variant gm.Product err error ) if !colorVariant.IsSellable { if err = importer.Baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil { log.Println(err) } continue } else { //todo check parsed import variant if variant, err = importer.importParsedVariant(colorVariant); err != nil { if variant, importer.Error = importer.updateParsedVariant(colorVariant); importer.Error != nil { return importer } linkedProducts = append(linkedProducts, variant) } else { newProducts = append(newProducts, variant) } } } if len(newProducts) > 0 { // relation var relation []gm.ProductRelation for _, linkedProduct := range linkedProducts { for _, newProd := range newProducts { relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID}) relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID}) } } if err := importer.Baza.Create(&relation).Error; err != nil { log.Println(err) } } } return importer } func (importer *Importer) UpdateOrCreate(product models.Product) (instance *Importer) { firstProduct, err := importer.updateVariant(product) var newProducts []gm.Product if err != nil { helper.Error(err) firstProduct, importer.Error = importer.updateVariant(product) if importer.Error != nil { return importer } } else if &firstProduct != nil { newProducts = append(newProducts, *firstProduct) } if product.ColorVariants != nil && len(*product.ColorVariants) > 0 { linkedProducts := []gm.Product{*firstProduct} for _, colorVariant := range *product.ColorVariants { var ( variant *gm.Product err error ) if !colorVariant.IsSellable { if err = importer.Baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil { log.Println(err) } continue } else { if variant, err = importer.importVariant(colorVariant); err != nil { if variant, importer.Error = importer.updateVariant(colorVariant); importer.Error != nil { return importer } linkedProducts = append(linkedProducts, *variant) } else { newProducts = append(newProducts, *variant) } } } if len(newProducts) > 0 { // relation var relation []gm.ProductRelation for _, linkedProduct := range linkedProducts { for _, newProd := range newProducts { relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID}) relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID}) } } if err := importer.Baza.Create(&relation).Error; err != nil { log.Println(err) } } } return importer }