package test import ( gm "db_service/gorm_models" "db_service/models" helper "db_service/pkg" "encoding/json" "fmt" "log" "os" "strconv" "sync" "gorm.io/driver/mysql" "gorm.io/gorm" ) var mainCategories []gm.Category var baza *gorm.DB var mainImportWG, famAndSellerWG sync.WaitGroup var families []gm.AttributeFamily var sellers []gm.MarketplaceSeller var attributesMap = make(map[string]gm.Attribute) func StartTest() error { baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{}) if err != nil { return err } //get categories from mysql on main thread mainCategories = gm.GetMainCategories(baza) famAndSellerWG.Add(3) //get attribute families from mysql with gorutine go func() { defer famAndSellerWG.Done() families = gm.GetFamilies(baza) }() //get sellers families from mysql with gorutine go func() { defer famAndSellerWG.Done() sellers = gm.GetSellers(baza) }() go func() { defer famAndSellerWG.Done() var attributes = gm.GetAttributes(baza) for _, atrattribute := range attributes { attributesMap[atrattribute.Code] = atrattribute } }() mainImportWG.Add(len(mainCategories)) for _, element := range mainCategories { slug := element.Translations[0].Slug go startImport("ty_db_"+slug, baza) // fmt.Println(<-result) } mainImportWG.Wait() fmt.Println("Terminating Program") return nil } func startImport(dbName string, db *gorm.DB) { defer mainImportWG.Done() dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName) fmt.Println(dbName) if dbExists { totalDocCount := getTotalDocumentCount(dbName) skip := 0 limit := 100 for skip < totalDocCount { var response models.BagistoModelResponse url := fmt.Sprintf("%s%s/_all_docs?include_docs=true&limit=%v&skip=%v", os.Getenv("couch_db_source"), dbName, limit, skip) fmt.Println(url) skip += limit body, err := helper.SendRequest("GET", url, nil, "", true) if err != nil { fmt.Println(err.Error()) continue } err = json.Unmarshal(body, &response) if err != nil { log.Println(err.Error()) continue } //itearate 100 row products for _, element := range response.Rows { importProduct(element.Doc, db) } } } else { fmt.Println(dbName + "+doesnt exist") } } func getTotalDocumentCount(db string) int { var response models.DBDocCountResponse url := os.Getenv("couch_db_source") + db body, err := helper.SendRequest("GET", url, nil, "", true) if err != nil { log.Println(err.Error()) return 0 } err = json.Unmarshal(body, &response) if err != nil { log.Println(err.Error()) return 0 } return response.DocCount } func importProduct(product models.Product, db *gorm.DB) { famAndSellerWG.Wait() //wait until attribute families and sellers are not get from mysql var categories []gm.Category var brand gm.Brand errCat := db.Find(&categories, product.Categories).Error if errCat != nil { log.Println(errCat) return } if product.Brand != "" { brand = gm.FindOrCreateBrand(db, product.Brand, categories) } iproduct := gm.Product{ Sku: product.ProductGroupID, Type: "simple", AttributeFamilyID: families[0].ID, Brand: brand, } for _, element := range product.Images { iproduct.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element}) } if product.ColorVariantCount > 0 || len(product.SizeVariants) > 0 { iproduct.Type = "configurable" iproduct.AttributeFamilyID = families[1].ID } err := db.Select("Sku", "Type", "AttributeFamilyID", "CreatedAt", "UpdatedAt").Create(&iproduct).Error if err != nil { log.Println(err) return } // mainProductFlat := gm.ProductFlat{} mainProductAttributeValues := assignAttributes(iproduct.ID, &product) if iproduct.Type == "simple" { mainProductAttributeValues = append(mainProductAttributeValues, getPriceAttributes(iproduct.ID, &product.Price)...) } else if len(product.ColorVariants) > 0 || len(product.SizeVariants) > 0 { var productVariants []gm.Product if len(product.ColorVariants) > 0 { //attach super attributes color superAttribute := gm.ProductSuperAttribute{ProductID: iproduct.ID, AttributeID: attributesMap["color"].ID} db.Create(&superAttribute) var description string for _, colorVariant := range product.ColorVariants { weight, _ := strconv.ParseFloat(colorVariant.Weight, 64) for _, desc := range colorVariant.Descriptions { description += "
" + desc.Description + "
" } colorOption := gm.GetAttributeOption(db, attributesMap["color"].ID, colorVariant.Color) if len(colorVariant.SizeVariants) > 0 { for _, sizeVariant := range colorVariant.SizeVariants { if sizeVariant.Sellable { sku := fmt.Sprintf("%s-%s-%d-col-size", colorVariant.ProductGroupID, colorVariant.ProductNumber, sizeVariant.ItemNumber) sizeOption := gm.GetAttributeOption(db, attributesMap["size"].ID, sizeVariant.AttributeValue) attributes := []gm.ProductAttributeValue{ {AttributeID: attributesMap["sku"].ID, TextValue: sku, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["product_number"].ID, TextValue: fmt.Sprintf("%s-%d", colorVariant.ProductNumber, sizeVariant.ItemNumber), Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["name"].ID, TextValue: colorVariant.Name, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"}, // {AttributeID: attributesMap["source"].ID, TextValue: colorVariant.URLKey, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["url_key"].ID, TextValue: sku, Channel: "default", Locale: "tm"}, // {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: colorVariant.FavoriteCount, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, { AttributeID: attributesMap["color"].ID, IntegerValue: int(colorOption.ID), Channel: "default", Locale: "tm", }, { AttributeID: attributesMap["size"].ID, IntegerValue: int(sizeOption.ID), Channel: "default", Locale: "tm", }, } if sizeVariant.Price.OriginalPrice.Value > sizeVariant.Price.DiscountedPrice.Value { attributes = append(attributes, []gm.ProductAttributeValue{ {AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["special_price"].ID, FloatValue: sizeVariant.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"}, }...) } else { attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}) } productVariant := gm.Product{ ParentID: iproduct.ID, Type: "simple", AttributeFamily: iproduct.AttributeFamily, Sku: sku, Brand: iproduct.Brand, AttributeValues: attributes, AttributeFamilyID: iproduct.AttributeFamilyID, } for _, element := range colorVariant.Images { productVariant.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element}) } //todo assign images productVariants = append(productVariants, productVariant) } } } else if colorVariant.IsSellable { colorOption := gm.GetAttributeOption(db, attributesMap["color"].ID, colorVariant.Color) attributes := collectAttributes(&colorVariant, &colorOption) if colorVariant.Price.OriginalPrice.Value > colorVariant.Price.DiscountedPrice.Value { attributes = append(attributes, []gm.ProductAttributeValue{ {AttributeID: attributesMap["price"].ID, FloatValue: colorVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["special_price"].ID, FloatValue: colorVariant.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"}, }...) } else { attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: colorVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}) } sku := fmt.Sprintf("%s-%s-col", iproduct.Sku, colorVariant.ProductNumber) productVariant := gm.Product{ ParentID: iproduct.ID, Type: "simple", AttributeFamily: iproduct.AttributeFamily, Sku: sku, Brand: iproduct.Brand, AttributeValues: attributes, AttributeFamilyID: iproduct.AttributeFamilyID, } for _, element := range colorVariant.Images { productVariant.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element}) } productVariants = append(productVariants, productVariant) } } } if len(product.SizeVariants) > 0 { //attach super attributes size superAttribute := gm.ProductSuperAttribute{ProductID: iproduct.ID, AttributeID: attributesMap["size"].ID} db.Create(&superAttribute) var description string for _, desc := range product.Descriptions { description += "" + desc.Description + "
" } weight, _ := strconv.ParseFloat(product.Weight, 64) for _, sizeVariant := range product.SizeVariants { if sizeVariant.Sellable { sku := fmt.Sprintf("%s-%s-%d-size", iproduct.Sku, product.ProductNumber, sizeVariant.ItemNumber) sizeOption := gm.GetAttributeOption(db, attributesMap["size"].ID, sizeVariant.AttributeValue) attributes := []gm.ProductAttributeValue{ {AttributeID: attributesMap["sku"].ID, TextValue: sku, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["product_number"].ID, TextValue: string(sizeVariant.ItemNumber), Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["name"].ID, TextValue: product.Name, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"}, // {AttributeID: attributesMap["source"].ID, TextValue: product.URLKey, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["url_key"].ID, TextValue: sku, Channel: "default", Locale: "tm"}, // {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: product.FavoriteCount, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, { AttributeID: attributesMap["size"].ID, IntegerValue: int(sizeOption.ID), Channel: "default", Locale: "tm", }, } if sizeVariant.Price.OriginalPrice.Value > sizeVariant.Price.DiscountedPrice.Value { attributes = append(attributes, []gm.ProductAttributeValue{ {AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["special_price"].ID, FloatValue: sizeVariant.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"}, }...) } else { attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}) } productVariant := gm.Product{ ParentID: iproduct.ID, Type: "simple", AttributeFamily: iproduct.AttributeFamily, Sku: sku, Brand: iproduct.Brand, AttributeValues: attributes, AttributeFamilyID: iproduct.AttributeFamilyID, } // for _, element := range colorVariant.Images { // productVariant.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element}) // } productVariants = append(productVariants, productVariant) } } } db.Create(&productVariants) //todo assign all with categories } db.Model(&iproduct).Association("Categories").Append(categories) var err2 = db.Create(&mainProductAttributeValues).Error if err2 != nil { log.Println(err) } // panic("'hassiktir'") //todo gorm transactions //todo Attributes source, cinsiyet // if len(product.Attributes) > 0 { // iproduct.AttrbuteFamily = getAttributeFamily() // } //todo seller product, meta keys // if($data['vendor'] && $seller = $this->vendorRepository->findOneByField('url',$data['vendor'])){ // $this->createSellerProduct($parentProduct, $seller->id); // } if product.Vendor != "" { for i := range sellers { if sellers[i].Url == product.Vendor { createSellerProduct() break } } } } func createSellerProduct() { /* Event::dispatch('marketplace.catalog.product.create.before'); $sellerProduct = $this->vendorProductRepository->create([ 'marketplace_seller_id' => $seller_id, 'is_approved' => 1, 'condition' => 'new', 'description' => 'scraped product', 'is_owner' => 1, 'product_id' => $product->id, ]); foreach ($sellerProduct->product->variants as $baseVariant) { $this->vendorProductRepository->create([ 'parent_id' => $sellerProduct->id, 'product_id' => $baseVariant->id, 'is_owner' => 1, 'marketplace_seller_id' => $seller_id, 'is_approved' => 1, 'condition' => 'new', ]); } Event::dispatch('marketplace.catalog.product.create.after', $sellerProduct); return $sellerProduct; */ } func collectAttributes(variant *models.Product, option *gm.AttributeOption) []gm.ProductAttributeValue { sku := fmt.Sprintf("%s-%s", variant.ProductGroupID, variant.ProductNumber) weight, _ := strconv.ParseFloat(variant.Weight, 64) var description string for _, desc := range variant.Descriptions { description += "" + desc.Description + "
" } return []gm.ProductAttributeValue{ {AttributeID: attributesMap["sku"].ID, TextValue: sku, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["product_number"].ID, TextValue: variant.ProductNumber, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["name"].ID, TextValue: variant.Name, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"}, // {AttributeID: attributesMap["source"].ID, TextValue: variant.URLKey, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["url_key"].ID, TextValue: sku, Channel: "default", Locale: "tm"}, // {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: variant.FavoriteCount, Channel: "default", Locale: "tm"}, {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, {AttributeID: option.AttributeID, IntegerValue: int(option.ID), Channel: "default", Locale: "tm"}, } } func assignAttributes(productID uint, data *models.Product) []gm.ProductAttributeValue { weight, _ := strconv.ParseFloat(data.Weight, 64) var description string for _, desc := range data.Descriptions { description += "" + desc.Description + "
" } //$desc = implode(array_map(fn($value): string => '' . $value['description'] . '
', $data['descriptions'])); var productAttributeValues = []gm.ProductAttributeValue{ {ProductID: productID, AttributeID: attributesMap["sku"].ID, TextValue: data.ProductGroupID, Channel: "default", Locale: "tm"}, {ProductID: productID, AttributeID: attributesMap["product_number"].ID, TextValue: data.ProductNumber, Channel: "default", Locale: "tm"}, {ProductID: productID, AttributeID: attributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"}, {ProductID: productID, AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"}, // {ProductID: productID, AttributeID: attributesMap["source"].ID, TextValue: data.URLKey, Channel: "default", Locale: "tm"}, {ProductID: productID, AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {ProductID: productID, AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"}, {ProductID: productID, AttributeID: attributesMap["url_key"].ID, TextValue: data.ProductGroupID, Channel: "default", Locale: "tm"}, // {ProductID: productID, AttributeID: attributesMap["short_description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, {ProductID: productID, AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, // {ProductID: productID, AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount, Channel: "default", Locale: "tm"}, } // if data.Price.OriginalPrice.Value > data.Price.DiscountedPrice.Value { // productAttributeValues = append(productAttributeValues, []gm.ProductAttributeValue{ // {ProductID: productID, AttributeID: attributesMap["price"].ID, FloatValue: data.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}, // {ProductID: productID, AttributeID: attributesMap["special_price"].ID, FloatValue: data.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"}, // }...) // } else { // productAttributeValues = append(productAttributeValues, gm.ProductAttributeValue{ProductID: productID, AttributeID: attributesMap["price"].ID, FloatValue: data.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"}) // } return productAttributeValues } func getPriceAttributes(productID uint, price *models.Price) []gm.ProductAttributeValue { var productAttributeValues []gm.ProductAttributeValue if price.OriginalPrice.Value > price.DiscountedPrice.Value { productAttributeValues = []gm.ProductAttributeValue{ {ProductID: productID, AttributeID: attributesMap["price"].ID, FloatValue: price.OriginalPrice.Value, Channel: "default", Locale: "tm"}, {ProductID: productID, AttributeID: attributesMap["special_price"].ID, FloatValue: price.DiscountedPrice.Value, Channel: "default", Locale: "tm"}, } } else { productAttributeValues = []gm.ProductAttributeValue{{ProductID: productID, AttributeID: attributesMap["price"].ID, FloatValue: price.OriginalPrice.Value, Channel: "default", Locale: "tm"}} } return productAttributeValues } func getAttributeFamily() gm.AttributeFamily { return families[0] }