package test import ( gm "db_service/gorm_models" "db_service/models" helper "db_service/pkg" "encoding/json" "fmt" "log" "os" "strconv" "sync" "gorm.io/driver/mysql" "gorm.io/gorm" ) var mainCategories []gm.Category var baza *gorm.DB var mainImportWG, famAndSellerWG sync.WaitGroup var families []gm.AttributeFamily var sellers = make(map[string]gm.MarketplaceSeller) var attributesMap = make(map[string]gm.Attribute) func StartTest() error { baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{}) if err != nil { return err } //get categories from mysql on main thread mainCategories = gm.GetMainCategories(baza) famAndSellerWG.Add(3) //get attribute families from mysql with gorutine go func() { defer famAndSellerWG.Done() families = gm.GetFamilies(baza) }() //get sellers families from mysql with gorutine go func() { defer famAndSellerWG.Done() for _, vendor := range gm.GetSellers(baza) { sellers[vendor.Url] = vendor } }() go func() { defer famAndSellerWG.Done() var attributes = gm.GetAttributes(baza) for _, atrattribute := range attributes { attributesMap[atrattribute.Code] = atrattribute } }() mainImportWG.Add(len(mainCategories)) for _, element := range mainCategories { slug := element.Translations[0].Slug go startImport("ty_db_"+slug, baza) // fmt.Println(<-result) } mainImportWG.Wait() fmt.Println("Terminating Program") return nil } func startImport(dbName string, db *gorm.DB) { defer mainImportWG.Done() dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName) fmt.Println(dbName) if dbExists { totalDocCount := getTotalDocumentCount(dbName) skip := 0 limit := 100 for skip < totalDocCount { var response models.BagistoModelResponse url := fmt.Sprintf("%s%s/_all_docs?include_docs=true&limit=%v&skip=%v", os.Getenv("couch_db_source"), dbName, limit, skip) fmt.Println(url) skip += limit body, err := helper.SendRequest("GET", url, nil, "", true) if err != nil { fmt.Println(err.Error()) continue } err = json.Unmarshal(body, &response) if err != nil { log.Println(err.Error()) continue } //itearate 100 row products for _, element := range response.Rows { importProduct(element.Doc, db) } } } else { fmt.Println(dbName + "+doesnt exist") } } func getTotalDocumentCount(db string) int { var response models.DBDocCountResponse url := os.Getenv("couch_db_source") + db body, err := helper.SendRequest("GET", url, nil, "", true) if err != nil { log.Println(err.Error()) return 0 } err = json.Unmarshal(body, &response) if err != nil { log.Println(err.Error()) return 0 } return response.DocCount } // todo defailt variant // productflat func getCats(db *gorm.DB, catIDs []int) ([]gm.Category, string, error) { var categories []gm.Category var keywords string errCat := db.Preload("Translations").Find(&categories, catIDs).Error if errCat != nil { return categories, keywords, errCat } for _, cat := range categories { if len(cat.Translations) > 0 && cat.Translations[0].MetaKeywords != "" { translation := cat.Translations[0] keywords += "," + translation.MetaKeywords } } return categories, keywords, nil } func importProduct(product models.Product, db *gorm.DB) { famAndSellerWG.Wait() //wait until attribute families and sellers are not get from mysql categories, keywords, errCat := getCats(db, product.Categories) if errCat != nil { log.Println(errCat) return } var brand gm.Brand if product.Brand != "" { brand = gm.FindOrCreateBrand(db, product.Brand, categories) } attributes, mainProductFlat := prepearAttributesWithFlat(&product) attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["meta_keywords"].ID, TextValue: keywords, Channel: "default", Locale: "tm"}) iproduct := gm.Product{ Sku: product.ProductGroupID, Type: "simple", AttributeFamilyID: families[0].ID, BrandID: brand.ID, Categories: categories, AttributeValues: attributes, } for _, element := range product.Images { iproduct.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element}) } if product.ColorVariantCount > 0 { iproduct.SuperAttributes = append(iproduct.SuperAttributes, attributesMap["color"]) } if len(product.SizeVariants) > 0 { iproduct.SuperAttributes = append(iproduct.SuperAttributes, attributesMap["size"]) } if len(iproduct.SuperAttributes) > 0 { iproduct.Type = "configurable" iproduct.AttributeFamilyID = families[1].ID } errMainProduct := db.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&iproduct).Error if errMainProduct != nil { log.Println(errMainProduct) return } mainProductFlat.ProductID = iproduct.ID mainProductFlat.BrandID = brand.ID mainProductFlat.MetaKeywords = keywords errProductMainFlat := db.Create(&mainProductFlat).Error if errProductMainFlat != nil { log.Println(errProductMainFlat) return } if iproduct.Type == "configurable" { var productFlatVariants []gm.ProductFlat if len(product.ColorVariants) > 0 { var description string for _, colorVariant := range product.ColorVariants { weight, _ := strconv.ParseFloat(colorVariant.Weight, 64) for _, desc := range colorVariant.Descriptions { description += "

" + desc.Description + "

" } colorOption := gm.GetAttributeOption(db, attributesMap["color"].ID, colorVariant.Color) if len(colorVariant.SizeVariants) > 0 { for _, sizeVariant := range colorVariant.SizeVariants { if sizeVariant.Sellable { sku := fmt.Sprintf("%s-%s-%d-col-size", colorVariant.ProductGroupID, colorVariant.ProductNumber, sizeVariant.ItemNumber) sizeOption := gm.GetAttributeOption(db, attributesMap["size"].ID, sizeVariant.AttributeValue) productNumber := fmt.Sprintf("%s-%d", colorVariant.ProductNumber, sizeVariant.ItemNumber) attributes := []gm.ProductAttributeValue{ {AttributeID: attributesMap["source"].ID, TextValue: colorVariant.URLKey, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: colorVariant.FavoriteCount, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["meta_keywords"].ID, TextValue: keywords, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["sku"].ID, TextValue: sku, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["product_number"].ID, TextValue: productNumber, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["name"].ID, TextValue: colorVariant.Name, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["url_key"].ID, TextValue: sku, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["meta_keywords"].ID, TextValue: keywords, Channel: "default", Locale: "tm"}, // {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, { AttributeID: attributesMap["color"].ID, IntegerValue: int(colorOption.ID), Channel: "default", Locale: "tm", }, { AttributeID: attributesMap["size"].ID, IntegerValue: int(sizeOption.ID), Channel: "default", Locale: "tm", }, } productVariant := gm.Product{ ParentID: mainProductFlat.ProductID, Type: "simple", AttributeFamilyID: iproduct.AttributeFamilyID, Sku: sku, BrandID: mainProductFlat.BrandID, AttributeValues: attributes, Categories: categories, } for _, element := range colorVariant.Images { productVariant.Images = append(productVariant.Images, gm.ProductImage{Type: "cdn", Path: element}) } errProdVariant := db.Omit("Categories.*").Create(&productVariant).Error if errProdVariant != nil { log.Println(errProdVariant) return } //todo assign images flatVariant := gm.ProductFlat{ ProductID: productVariant.ID, BrandID: mainProductFlat.BrandID, Status: true, VisibleIndividually: true, Name: colorVariant.Name, Sku: sku, ProductNumber: productNumber, Description: description, UrlKey: sku, Weight: weight, // Source: colorVariant.URLKey, FavoritesCount: uint(colorVariant.FavoriteCount), Color: int(colorOption.ID), Size: int(sizeOption.ID), ColorLabel: colorOption.AdminName, SizeLabel: sizeOption.AdminName, MetaKeywords: keywords, ParentID: &mainProductFlat.ID, } if sizeVariant.Price.OriginalPrice.Value > sizeVariant.Price.DiscountedPrice.Value { attributes = append(attributes, []gm.ProductAttributeValue{ {AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["special_price"].ID, FloatValue: sizeVariant.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"}, }...) flatVariant.Price = sizeVariant.Price.OriginalPrice.Value flatVariant.SpecialPrice = sizeVariant.Price.DiscountedPrice.Value flatVariant.MinPrice = sizeVariant.Price.DiscountedPrice.Value flatVariant.MaxPrice = sizeVariant.Price.OriginalPrice.Value } else { attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}) flatVariant.Price = sizeVariant.Price.OriginalPrice.Value flatVariant.MinPrice = sizeVariant.Price.DiscountedPrice.Value flatVariant.MaxPrice = sizeVariant.Price.OriginalPrice.Value } if mainProductFlat.MinPrice > flatVariant.MinPrice { mainProductFlat.MinPrice = flatVariant.MinPrice } if mainProductFlat.MaxPrice < flatVariant.MaxPrice { mainProductFlat.MaxPrice = flatVariant.MaxPrice } errVariant := db.Create(&flatVariant).Error if errVariant != nil { log.Println(errVariant) return } productFlatVariants = append(productFlatVariants, flatVariant) } } } else if colorVariant.IsSellable { colorOption := gm.GetAttributeOption(db, attributesMap["color"].ID, colorVariant.Color) attributes, variantFlat := collectAttributes(&colorVariant, &colorOption) attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["meta_keywords"].ID, TextValue: keywords, Channel: "default", Locale: "tm"}) if mainProductFlat.MinPrice > variantFlat.MinPrice { mainProductFlat.MinPrice = variantFlat.MinPrice } if mainProductFlat.MaxPrice < variantFlat.MaxPrice { mainProductFlat.MaxPrice = variantFlat.MaxPrice } sku := fmt.Sprintf("%s-%s-%s", iproduct.Sku, colorVariant.ProductNumber, colorVariant.ProductCode) productVariant := gm.Product{ ParentID: mainProductFlat.ProductID, Type: "simple", Sku: sku, BrandID: mainProductFlat.BrandID, AttributeValues: attributes, AttributeFamilyID: iproduct.AttributeFamilyID, Categories: categories, } for _, element := range colorVariant.Images { productVariant.Images = append(productVariant.Images, gm.ProductImage{Type: "cdn", Path: element}) } errProdVariant := db.Omit("Categories.*").Create(&productVariant).Error if errProdVariant != nil { log.Println(errProdVariant) return } variantFlat.ProductID = productVariant.ID variantFlat.Color = int(colorOption.ID) variantFlat.ColorLabel = colorOption.AdminName variantFlat.MetaKeywords = keywords variantFlat.ParentID = &mainProductFlat.ID errVariant := db.Create(&variantFlat).Error if errVariant != nil { log.Println(errVariant) return } productFlatVariants = append(productFlatVariants, variantFlat) } } } if len(product.SizeVariants) > 0 { var description string for _, desc := range product.Descriptions { description += "

" + desc.Description + "

" } weight, _ := strconv.ParseFloat(product.Weight, 64) for _, sizeVariant := range product.SizeVariants { if sizeVariant.Sellable { sku := fmt.Sprintf("%s-%s-%d-size", iproduct.Sku, product.ProductNumber, sizeVariant.ItemNumber) sizeOption := gm.GetAttributeOption(db, attributesMap["size"].ID, sizeVariant.AttributeValue) attributes := []gm.ProductAttributeValue{ {AttributeID: attributesMap["source"].ID, TextValue: product.URLKey}, {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: product.FavoriteCount}, {AttributeID: attributesMap["meta_keywords"].ID, TextValue: keywords, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["sku"].ID, TextValue: sku}, {AttributeID: attributesMap["product_number"].ID, TextValue: fmt.Sprint(sizeVariant.ItemNumber)}, {AttributeID: attributesMap["name"].ID, TextValue: product.Name, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["weight"].ID, FloatValue: weight}, {AttributeID: attributesMap["status"].ID, BooleanValue: true}, {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true}, {AttributeID: attributesMap["url_key"].ID, TextValue: sku}, {AttributeID: attributesMap["meta_keywords"].ID, TextValue: keywords, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["size"].ID, IntegerValue: int(sizeOption.ID)}, // {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, } flatVariant := gm.ProductFlat{ Status: true, VisibleIndividually: true, Name: product.Name, Sku: sku, ProductNumber: fmt.Sprint(sizeVariant.ItemNumber), UrlKey: sku, Weight: weight, Size: int(sizeOption.ID), SizeLabel: sizeOption.AdminName, MetaKeywords: keywords, Product: gm.Product{ ParentID: mainProductFlat.ProductID, Type: "simple", Sku: sku, BrandID: mainProductFlat.BrandID, AttributeValues: attributes, AttributeFamilyID: iproduct.AttributeFamilyID, Categories: categories, }, // Source: product.URLKey, FavoritesCount: uint(product.FavoriteCount), ParentID: &mainProductFlat.ID, } if sizeVariant.Price.OriginalPrice.Value > sizeVariant.Price.DiscountedPrice.Value { attributes = append(attributes, []gm.ProductAttributeValue{ {AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value}, {AttributeID: attributesMap["special_price"].ID, FloatValue: sizeVariant.Price.DiscountedPrice.Value}, }...) flatVariant.Price = sizeVariant.Price.OriginalPrice.Value flatVariant.SpecialPrice = sizeVariant.Price.DiscountedPrice.Value flatVariant.MinPrice = sizeVariant.Price.DiscountedPrice.Value flatVariant.MaxPrice = sizeVariant.Price.OriginalPrice.Value } else { attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value}) flatVariant.Price = sizeVariant.Price.OriginalPrice.Value flatVariant.MinPrice = sizeVariant.Price.DiscountedPrice.Value flatVariant.MaxPrice = sizeVariant.Price.OriginalPrice.Value } if mainProductFlat.MinPrice > flatVariant.MinPrice { mainProductFlat.MinPrice = flatVariant.MinPrice } if mainProductFlat.MaxPrice < flatVariant.MaxPrice { mainProductFlat.MaxPrice = flatVariant.MaxPrice } errVariant := db.Omit("Product.Categories.*").Create(&flatVariant).Error if errVariant != nil { log.Println(errVariant) return } productFlatVariants = append(productFlatVariants, flatVariant) } } } errFlat := db.Omit("Product.ParentID", "Product.SuperAttributes.*", "Product.Categories.*", "Source", "FavoritesCount", "CreatedAt").Save(&mainProductFlat) if errFlat != nil { log.Println(errFlat) } mainProductFlat.Variants = productFlatVariants } sProduct := createSellerProduct(&mainProductFlat, product.Vendor) //todo seller product, meta keys errSProduct := db.Create(&sProduct).Error if errSProduct != nil { log.Println(errSProduct) } } func createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct { sellerProduct := gm.MarketplaceProduct{ MarketplaceSeller: sellers[sellerURL], IsApproved: true, Condition: "new", Description: "scraped", IsOwner: true, ProductID: flat.ProductID, } for _, variant := range flat.Variants { sellerProduct.Variants = append(sellerProduct.Variants, gm.MarketplaceProduct{ ProductID: variant.ProductID, IsOwner: true, IsApproved: true, MarketplaceSeller: sellers[sellerURL], Condition: "new", }) } return sellerProduct } func collectAttributes(variant *models.Product, option *gm.AttributeOption) ([]gm.ProductAttributeValue, gm.ProductFlat) { sku := fmt.Sprintf("%s-%s", variant.ProductGroupID, variant.ProductNumber) weight, _ := strconv.ParseFloat(variant.Weight, 64) // var description string // for _, desc := range variant.Descriptions { // description += "

" + desc.Description + "

" // } flat := gm.ProductFlat{ Status: true, VisibleIndividually: true, Name: variant.Name, Sku: sku, ProductNumber: variant.ProductNumber, UrlKey: sku, Weight: weight, // Source: variant.URLKey, // FavoritesCount: uint(variant.FavoriteCount), } attributes := []gm.ProductAttributeValue{ {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: variant.FavoriteCount}, {AttributeID: attributesMap["source"].ID, TextValue: variant.URLKey}, {AttributeID: attributesMap["sku"].ID, TextValue: sku}, {AttributeID: attributesMap["product_number"].ID, TextValue: variant.ProductNumber}, {AttributeID: attributesMap["name"].ID, TextValue: variant.Name, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["weight"].ID, FloatValue: weight}, {AttributeID: attributesMap["status"].ID, BooleanValue: true}, {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true}, {AttributeID: attributesMap["url_key"].ID, TextValue: sku}, {AttributeID: option.AttributeID, IntegerValue: int(option.ID)}, // {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, } if variant.Price.OriginalPrice.Value > variant.Price.DiscountedPrice.Value { attributes = append(attributes, []gm.ProductAttributeValue{ {AttributeID: attributesMap["price"].ID, FloatValue: variant.Price.OriginalPrice.Value}, {AttributeID: attributesMap["special_price"].ID, FloatValue: variant.Price.DiscountedPrice.Value}, }...) flat.Price = variant.Price.OriginalPrice.Value flat.SpecialPrice = variant.Price.DiscountedPrice.Value flat.MinPrice = variant.Price.DiscountedPrice.Value flat.MaxPrice = variant.Price.OriginalPrice.Value } else { attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: variant.Price.OriginalPrice.Value}) flat.Price = variant.Price.OriginalPrice.Value flat.MinPrice = variant.Price.DiscountedPrice.Value flat.MaxPrice = variant.Price.OriginalPrice.Value } return attributes, flat } func prepearAttributesWithFlat(data *models.Product) ([]gm.ProductAttributeValue, gm.ProductFlat) { weight, _ := strconv.ParseFloat(data.Weight, 64) var description string for _, desc := range data.Descriptions { description += "

" + desc.Description + "

" } //$desc = implode(array_map(fn($value): string => '

' . $value['description'] . '

', $data['descriptions'])); var productAttributeValues = []gm.ProductAttributeValue{ {AttributeID: attributesMap["source"].ID, TextValue: data.URLKey}, {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount}, {AttributeID: attributesMap["sku"].ID, TextValue: data.ProductGroupID}, {AttributeID: attributesMap["product_number"].ID, TextValue: data.ProductNumber}, {AttributeID: attributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["weight"].ID, FloatValue: weight}, {AttributeID: attributesMap["status"].ID, BooleanValue: true}, {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true}, {AttributeID: attributesMap["url_key"].ID, TextValue: data.ProductGroupID}, {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, } flat := gm.ProductFlat{ Status: true, VisibleIndividually: true, Name: data.Name, Sku: data.ProductGroupID, ProductNumber: data.ProductNumber, Description: description, UrlKey: data.ProductGroupID, Weight: weight, // Source: data.URLKey, FavoritesCount: uint(data.FavoriteCount), } if data.Price.OriginalPrice.Value > data.Price.DiscountedPrice.Value { productAttributeValues = append(productAttributeValues, []gm.ProductAttributeValue{ {AttributeID: attributesMap["price"].ID, FloatValue: data.Price.OriginalPrice.Value}, {AttributeID: attributesMap["special_price"].ID, FloatValue: data.Price.DiscountedPrice.Value}, }...) flat.Price = data.Price.OriginalPrice.Value flat.SpecialPrice = data.Price.DiscountedPrice.Value flat.MinPrice = data.Price.DiscountedPrice.Value flat.MaxPrice = data.Price.OriginalPrice.Value } else { productAttributeValues = append(productAttributeValues, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: data.Price.OriginalPrice.Value}) flat.Price = data.Price.OriginalPrice.Value flat.MinPrice = data.Price.OriginalPrice.Value flat.MaxPrice = data.Price.OriginalPrice.Value } if data.ColorVariantCount != 0 || len(data.SizeVariants) != 0 { flat.Price = 0 flat.SpecialPrice = 0 } return productAttributeValues, flat }