Merge branch 'parse'

This commit is contained in:
merdan 2023-09-22 18:19:27 +05:00
commit 9b6a83d866
23 changed files with 1523 additions and 170 deletions

View File

@ -1,17 +1,12 @@
couch_db_source=http://admin:adminadmin@192.168.1.2:5984/
couch_db_source=
db_trendyol=trendyol_db_v2
db_ty_categories=ty_categories
remote_url=https://sarga.com.tm/api/
remote_url=
descendant_category_url=descendant-categories
db=test_baza
du=merdan
dp=qazwsx12
da=localhost
dp=3306
#database_url ="merdan:qazwsx12@tcp(127.0.0.1:3306)/test_baza?parseTime=true"
database_url ="orient:orient@tcp(192.168.1.2:3306)/bagisto_sarga?parseTime=true"
couchdb_url = "http://admin:adminadmin@192.168.1.2:5984/"
scout_flash = "http://sarga.com.tm/scrap/scoute-flush"
database_url=
couchdb_url=
scout_flash=
port=

View File

@ -1065,36 +1065,3 @@ func UpdateProduct(product models.Product, db *gorm.DB, productFlat gm.ProductFl
return errFlat
}
//func productAttributesAndFlat(data *models.Product) ([]gm.ProductAttributeValue,gm.ProductFlat){
//
// var description string
//
// for _, desc := range data.Descriptions {
// description += "<p>" + desc.Description + "</p>"
// }
//
// weight, _ := strconv.ParseFloat(data.Weight, 64)
//
// flat := gm.ProductFlat{
// Status: true,
// VisibleIndividually: true,
// Name: data.Name,
// Sku: data.ProductGroupID,
// ProductNumber: data.ProductNumber,
// Description: description,
// UrlKey: data.ProductGroupID,
// Weight: weight,
// FavoritesCount: uint(data.FavoriteCount),
// }
// return []gm.ProductAttributeValue{
// {AttributeID: AttributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount},
// {AttributeID: AttributesMap["source"].ID, TextValue: data.URLKey},
// {AttributeID: AttributesMap["product_number"].ID, TextValue: data.ProductNumber},
// {AttributeID: AttributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"},
// {AttributeID: AttributesMap["weight"].ID, TextValue: data.Weight},
// {AttributeID: AttributesMap["status"].ID, BooleanValue: true},
// {AttributeID: AttributesMap["visible_individually"].ID, BooleanValue: true},
// {AttributeID: AttributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
// },flat
//}

View File

@ -46,6 +46,7 @@ func StartProductImport(w http.ResponseWriter, _ *http.Request) {
}
importRepo.ImportWGroup.Wait()
//todo delete galan wishlist
//scout index flush
if err = gm.Flush(); err != nil {

View File

@ -1,29 +1,82 @@
package controller
import (
gm "db_service/gorm_models"
"db_service/models"
helper "db_service/pkg"
"db_service/repositories"
"encoding/json"
"errors"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"fmt"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
)
func ParseLink(w http.ResponseWriter, route *http.Request) {
start := time.Now()
link := route.URL.Query().Get("url")
helper.Info("link: ", link)
if strings.Contains(link, "lcwaikiki.com"){
ParseLinkLCW(link,w)
}else {
ParseLinkTY(link,w)
}
}
func ParseLinkLCW(link string,w http.ResponseWriter) {
start := time.Now()
fmt.Println(link)
linkParser := repositories.NewLCWScraper(link)
product, err := linkParser.InitProductDetailParsing()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]string{
"msg": err.Error(),
})
fmt.Println(err)
return
}
/////////////////////////////
importRepo, err := repositories.ParseImporterInstance()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
fmt.Println(err)
return
}
//wait until initialization data is loaded to memory
importRepo.ImportWGroup.Wait()
log.Println(product);
if err = importRepo.UpdateOrCreateLCW(product).Error; err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
fmt.Println(err)
return
}
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]string{
"msg": "Link parsed successfully",
"productGroupId": product.ProductNumber,
})
elapsed := time.Since(start)
log.Printf("end parse took %s", elapsed)
return
}
func ParseLinkTY(link string,w http.ResponseWriter) {
start := time.Now()
linkParser := repositories.NewLinkParser(link)
//todo
product, err := linkParser.ParseLink()
if err != nil {
@ -31,70 +84,75 @@ func ParseLink(w http.ResponseWriter, route *http.Request) {
json.NewEncoder(w).Encode(map[string]string{
"msg": err.Error(),
})
return
}
jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID)
if err != nil {
helper.Error(err)
return
}
baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{})
if err != nil {
helper.Error(err)
return
}
var productFlat gm.ProductFlat
err = baza.Preload("Variants").
//Preload("Product").
//Preload("Proudct.AttributeValues","attribute_id in(11,13)").
First(&productFlat, "sku = ?", jsonProduct.ProductGroupID).Error
if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
helper.Error(err)
if len(AttributesMap) == 0 {
var attributes, err = gm.GetAttributes(baza)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
for _, atrattribute := range attributes {
AttributesMap[atrattribute.Code] = atrattribute
}
}
err = ImportProduct(jsonProduct, baza)
} else if err == nil {
err = UpdateProduct(jsonProduct, baza, productFlat)
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]string{
"msg": "Link parsed successfully",
"productGroupId": strconv.Itoa(product.ProductGroupID),
})
elapsed := time.Since(start)
log.Printf("end parse took %s", elapsed)
return
}
if err != nil {
helper.Error(err)
fmt.Println(err)
return
}
elapsed := time.Since(start)
log.Printf("end parse took %s", elapsed)
log.Printf("link parsed at %s", elapsed)
jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID)
elapsed = time.Since(start)
log.Printf("product gat at %s", elapsed)
if err != nil {
helper.Error(err)
return
}
/////////////////////////////
importRepo, err := repositories.ImporterInstance()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//wait until initialization data is loaded to memory
importRepo.ImportWGroup.Wait()
if err = importRepo.UpdateOrCreate(jsonProduct).Error; err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]string{
"msg": "Link parsed successfully",
"productGroupId": strconv.Itoa(product.ProductGroupID),
"productGroupId": strconv.Itoa(product.ID),
})
elapsed = time.Since(start)
log.Printf("imported at %s", elapsed)
return
/////////////////////////////////////////
}
func ImportLink(w http.ResponseWriter, route *http.Request) {
var product models.Product
err := json.NewDecoder(route.Body).Decode(&product)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
/////////////////////////////
importRepo, err := repositories.ImporterInstance()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//wait until initialization data is loaded to memory
importRepo.ImportWGroup.Wait()
if err = importRepo.UpdateOrCreate(product).Error; err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]string{
"msg": "Link parsed successfully",
"productGroupId": product.ProductGroupID,
})
}

12
go.mod
View File

@ -3,6 +3,9 @@ module db_service
go 1.19
require (
github.com/PuerkitoBio/goquery v1.8.0
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24
github.com/chromedp/chromedp v0.8.7
github.com/gorilla/mux v1.8.0
github.com/gosimple/slug v1.12.0
github.com/joho/godotenv v1.4.0
@ -12,8 +15,17 @@ require (
)
require (
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.1.0 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
)

32
go.sum
View File

@ -1,5 +1,21 @@
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24 h1:IKBU366KowxMcDpu9cgYEaXsiO0eBfPosTS0Krqfb8M=
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
github.com/chromedp/chromedp v0.8.7 h1:dYOYc5ynTBzwSLOi+1IfgHwPr8r2BqV48l/RC+3OuJ0=
github.com/chromedp/chromedp v0.8.7/go.mod h1:iL+ywnwk3eG3EVXV1ackXBMNzdEh3Ye/KHvQkq1KRKU=
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gosimple/slug v1.12.0 h1:xzuhj7G7cGtd34NXnW/yF0l+AGNfWqwgh/IXgFy7dnc=
@ -13,8 +29,24 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/leesper/couchdb-golang v1.2.1 h1:FqSaTxxT2mVRLbxGQVkZakRRoSzWhPmV8UEKYjA/GWc=
github.com/leesper/couchdb-golang v1.2.1/go.mod h1:OU3FDAM3mazHx15oi8Hm+egTMneBUqepwnh0LuBSH54=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY=
gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=

View File

@ -26,14 +26,21 @@ func FindOrCreateBrand(db *gorm.DB, brand string, categories []Category) (Brand,
code := slug.Make(brand)
err := db.Omit("Categories").FirstOrCreate(&brandObject, Brand{Name: brand, Code: code, Categories: categories}).Error
// err := db.Model(&Brand{}).Find(&brandObject).Error
err := db.FirstOrCreate(&brandObject, Brand{Name: brand, Code: code}).Error
//err := db.Model(Brand{Name: brand, Code: code}).First(&brandObject).Error
//log.Println(Brand{Name: brand, Code: code})
if err != nil {
//err := db.Omit("Categories.*").FirstOrCreate(&brandObject, Brand{Name: brand, Code: code, Categories: categories}).Error
log.Println("ERR0000000000000001" + err.Error())
return brandObject, err
}
db.Model(&brandObject).Association("Categories").Append(categories)
return brandObject, nil
}
// var brandObject Brand

View File

@ -1,16 +1,25 @@
package gorm_models
import (
"errors"
"log"
"gorm.io/gorm"
)
type AttributeOption struct {
ID uint `gorm:"primaryKey"`
AttributeID uint
AdminName string
SortOrder int `sql:"DEFAULT:NULL" gorm:"default:null"`
ID uint `gorm:"primaryKey"`
AttributeID uint
AdminName string
SortOrder int `sql:"DEFAULT:NULL" gorm:"default:null"`
Translations []AttributeOptionTranslation
}
type AttributeOptionTranslation struct {
ID uint `gorm:"primaryKey"`
AttributeOptionID uint
Locale string
Label string
}
type Attribute struct {
@ -61,11 +70,27 @@ func GetAttributes(db *gorm.DB) ([]Attribute, error) {
func GetAttributeOption(db *gorm.DB, attrbuteID uint, value string) AttributeOption {
var option AttributeOption
err := db.FirstOrCreate(&option, AttributeOption{AttributeID: attrbuteID, AdminName: value}).Error
if err != nil {
log.Println(err.Error())
err := db.Where("attribute_id=? and admin_name=?", attrbuteID, value).First(&option).Error
if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
option = AttributeOption{
AttributeID: attrbuteID,
AdminName: value,
SortOrder: 1000,
Translations: []AttributeOptionTranslation{
{Locale: "tm", Label: value},
{Locale: "ru", Label: value},
{Locale: "tr", Label: value},
},
}
if err := db.Create(&option).Error; err != nil {
log.Println(err.Error())
}
}
return option
}

View File

@ -4,6 +4,7 @@ import (
"context"
helper "db_service/pkg"
"gorm.io/gorm"
"log"
"os"
"time"
)
@ -32,13 +33,12 @@ type ProductRelation struct {
}
type ProductFlat struct {
ID uint `gorm:"primary_key"`
Sku string
ProductNumber string `sql:"DEFAULT:NULL" gorm:"default:null"`
Name string `sql:"DEFAULT:''" gorm:"default:''"`
Weight float64 `sql:"DEFAULT:NULL" gorm:"type:decimal(12,4);default:null"`
Status bool `sql:"DEFAULT:NULL" gorm:"default:null"`
//Source string `sql:"DEFAULT:NULL" gorm:"default:null"`
ID uint `gorm:"primary_key"`
Sku string
ProductNumber string `sql:"DEFAULT:NULL" gorm:"default:null"`
Name string `sql:"DEFAULT:''" gorm:"default:''"`
Weight float64 `sql:"DEFAULT:NULL" gorm:"type:decimal(12,4);default:null"`
Status bool `sql:"DEFAULT:NULL" gorm:"default:null"`
VisibleIndividually bool `sql:"DEFAULT:NULL" gorm:"default:null"`
Price float64 `gorm:"type:decimal(12,4)"`
MinPrice float64 `gorm:"type:decimal(12,4)"`
@ -60,10 +60,14 @@ type ProductFlat struct {
ColorLabel string `sql:"DEFAULT:NULL" gorm:"default:null"`
Size int `sql:"DEFAULT:NULL" gorm:"default:null"`
SizeLabel string `sql:"DEFAULT:NULL" gorm:"default:null"`
Boyut int `sql:"DEFAULT:NULL" gorm:"default:null"`
BoyutLabel string `sql:"DEFAULT:NULL" gorm:"default:null"`
MetaTitle string `sql:"DEFAULT:NULL" gorm:"default:null"`
MetaKeywords string `sql:"DEFAULT:NULL" gorm:"default:null"`
BrandID uint `sql:"DEFAULT:NULL" gorm:"default:null"`
Brand Brand
Cinsiyet int `sql:"DEFAULT:NULL" gorm:"default:null"`
CinsiyetLabel string `sql:"DEFAULT:NULL" gorm:"default:null"`
}
type ProductImage struct {
@ -104,10 +108,43 @@ func DeleteProducts(db *gorm.DB) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
//qb := "DELETE FROM products WHERE id NOT IN (select product_id as id from wishlist) AND id NOT IN (select product_id as id from order_items) AND id NOT IN (select parent_id as idfrom order_items);"
qb := "DELETE p FROM products p LEFT JOIN order_items oi ON p.id = oi.product_id LEFT JOIN order_items op ON p.id = op.parent_id WHERE oi.id IS NULL AND op.id IS NULL;"
return db.WithContext(ctx).Exec(qb).Error
qb := "DELETE p FROM products p " +
"LEFT JOIN order_items oi ON p.id = oi.product_id " +
"LEFT JOIN order_items op ON p.id = op.parent_id " +
"LEFT JOIN wishlist wp ON p.id = wp.product_id " +
"LEFT JOIN wishlist wl ON wl.product_id = p.parent_id " +
"JOIN marketplace_products mp ON p.id = mp.product_id "+
"WHERE oi.id IS NULL AND op.id IS NULL AND wp.id IS NULL AND wl.id IS NULL AND mp.marketplace_seller_id=1;"
db.WithContext(ctx).Exec(qb)
//db.WithContext(ctx).Exec("UPDATE product_flat set sku=concat(id,\"-ordered\"), status=0 where status=1 AND product_id IN (SELECT product_id from marketplace_products where marketplace_seller_id=1)" )
//db.WithContext(ctx).Exec("UPDATE products set sku=concat(id,\"-ordered\") WHERE id IN (SELECT product_id from marketplace_products where marketplace_seller_id=1)")
return db.Error
}
func GetWishlistProducts(db *gorm.DB) ([]Product, error){
var products [] Product
err := db.Joins("JOIN wishlist wp ON products.id = wp.product_id").Joins("JOIN marketplace_products mp ON products.id = mp.product_id where marketplace_seller_id=1").Find(&products).Error
if err != nil {
log.Println(err.Error())
return nil, err
}
return products, nil
//qb := "SELECT p.id,p.sku FROM products p " +
// "JOIN wishlist wp ON p.id = wp.product_id " +
// "JOIN marketplace_products mp ON p.id = mp.product_id where marketplace_seller_id=1);"
}
//func DisableProducts (db *gorm.DB) error {
// ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
// defer cancel()
// db.WithContext(ctx).Exec("UPDATE product_flat set sku=concat(sku,\"-ordered\"), status=0")
// db.WithContext(ctx).Exec("UPDATE products set sku=concat(sku,\"-ordered\")")
// return db.Error
//}
func Flush() error {
_, err := helper.SendRequest("GET", os.Getenv("scout_flash"), nil, "")

View File

@ -11,6 +11,7 @@ type MarketplaceProduct struct {
CreatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"`
UpdatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"`
ProductID uint
//Product Product
ParentID *uint `sql:"DEFAULT:NULL" gorm:"default:null"`
Condition string `sql:"DEFAULT:NULL" gorm:"default:null"`
Price float64

11
gorm_models/wishlist.go Normal file
View File

@ -0,0 +1,11 @@
package gorm_models
import "time"
type Wishlist struct {
ID uint `gorm:"primaryKey"`
CreatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"`
UpdatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"`
ProductID uint
//Product Product
}

View File

@ -49,6 +49,7 @@ func main() {
//route.HandleFunc("/init-importer", controller.StartImport)
route.HandleFunc("/init-importer", controller.StartProductImport)
route.HandleFunc("/parse-link", controller.ParseLink)
route.HandleFunc("/import-link", controller.ImportLink).Methods("POST", "GET")
err := http.ListenAndServe(os.Getenv("port"), route)
if err != nil {

View File

@ -0,0 +1,17 @@
package models
type LCWScrapDataModel struct {
Id string
Code string
Name string
OldPrice string
Price string
SizeVariants []SizeVariants
ColorOptions []LCWScrapDataModel
}
type DescriptionModel struct {
Description string `json:"description"`
Bold bool `json:"bold"`
}

View File

@ -32,20 +32,16 @@ type Product struct {
}
type Price struct {
ProfitMargin int `json:"profitMargin"`
DiscountedPrice struct {
Text string `json:"text"`
Value float64 `json:"value"`
} `json:"discountedPrice"`
SellingPrice struct {
Text string `json:"text"`
Value float64 `json:"value"`
} `json:"sellingPrice"`
OriginalPrice struct {
Text string `json:"text"`
Value float64 `json:"value"`
} `json:"originalPrice"`
Currency string `json:"currency"`
ProfitMargin int `json:"profitMargin"`
DiscountedPrice PriceValue `json:"discountedPrice"`
SellingPrice PriceValue `json:"sellingPrice"`
OriginalPrice PriceValue `json:"originalPrice"`
Currency string `json:"currency"`
}
type PriceValue struct {
Text string `json:"text"`
Value float64 `json:"value"`
}
type Variant struct {
@ -163,7 +159,7 @@ type TrendyolProductDetailModel struct {
ProductCode string `json:"productCode"`
Name string `json:"name"`
NameWithProductCode string `json:"nameWithProductCode"`
Description string `json:"description"`
//Description string `json:"description"`
ContentDescriptions []struct {
Description string `json:"description"`
Bold bool `json:"bold"`

View File

@ -0,0 +1,7 @@
package models
type SizeVariants struct {
Price float64 `json:"price"`
Size string `json:"size"`
Stock int `json:"stock"`
}

View File

@ -2,6 +2,7 @@ package helper
import (
"context"
"db_service/models"
"log"
"net"
"net/http"
@ -84,3 +85,28 @@ func GetCategoryWeight(slug string) string {
return doc["weight"].(string)
}
func IsLCWSizeVariantsAdded(items []models.Variant, keyItem models.Variant) bool {
for _, option := range items {
if option.ItemNumber == keyItem.ItemNumber {
return true
}
}
return false
}
func IsImageAdded(images []string, image string) bool {
for _, img := range images {
if img == image {
return true
}
}
return false
}
func IsVariantsAdded(variants []models.Variant, itemNumber int) bool {
for _, variant := range variants {
if variant.ItemNumber == itemNumber {
return true
}
}
return false
}

View File

@ -12,8 +12,8 @@ import (
func SendRequest(method string, endpoint string, values []byte, authKey string) ([]byte, error) {
const ConnectMaxWaitTime = 30 * time.Second
const RequestMaxWaitTime = 60 * time.Second
const ConnectMaxWaitTime = time.Minute
const RequestMaxWaitTime = 5 * time.Minute
client := http.Client{
Transport: &http.Transport{

View File

@ -22,6 +22,7 @@ type Importer struct {
baza *gorm.DB
families []gm.AttributeFamily
sellers map[string]gm.MarketplaceSeller
//wishlist map[string]gm.Product
AttributesMap map[string]gm.Attribute
Error error
ImportWGroup sync.WaitGroup
@ -30,7 +31,10 @@ type Importer struct {
ColorMutex sync.Mutex
SexMutex sync.Mutex
}
type GormErr struct {
Number int `json:"Number"`
Message string `json:"Message"`
}
func ImporterInstance() (instance *Importer, err error) {
db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true})
@ -48,6 +52,7 @@ func ImporterInstance() (instance *Importer, err error) {
go func(db *gorm.DB) {
defer instance.ImportWGroup.Done()
instance.mainCategories, instance.Error = gm.GetMainCategories(db)
}(db)
//load families to memory
@ -113,6 +118,105 @@ func ImporterInstance() (instance *Importer, err error) {
}
}()
//load wishlist to memory
//go func(){
// defer instance.ImportWGroup.Done()
//
// var wishlist, err = gm.GetWishlistProducts(db)
//
// if err != nil {
// instance.Error = err
// return
// }
//
// instance.wishlist = make(map[string]gm.Product, len(wishlist))
//
// for _, product := range wishlist {
// instance.wishlist[product.Sku] = product
// }
//}()
if instance.Error != nil {
log.Println(instance.Error)
return nil, instance.Error
}
return instance, nil
}
func ParseImporterInstance() (instance *Importer, err error) {
db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true})
if err != nil {
log.Println(err)
return nil, err
}
instance = &Importer{baza: db}
instance.ImportWGroup.Add(3)
//load families to memory
go func() {
defer instance.ImportWGroup.Done()
instance.families, instance.Error = gm.GetFamilies(db)
}()
//load attributes to memory
go func() {
defer instance.ImportWGroup.Done()
if attributes, err := gm.GetAttributes(db); err != nil {
instance.Error = err
return
} else {
instance.AttributesMap = make(map[string]gm.Attribute, len(attributes))
for _, attribute := range attributes {
instance.AttributesMap[attribute.Code] = attribute
}
}
if colorOptions, err := gm.GetAttrOptions(db, instance.AttributesMap["color"].ID); err != nil {
instance.Error = err
return
} else {
instance.ColorOptions = make(map[string]gm.AttributeOption, len(colorOptions))
for _, option := range colorOptions {
instance.ColorOptions[option.AdminName] = option
}
}
if sexOPtions, err := gm.GetAttrOptions(db, instance.AttributesMap["cinsiyet"].ID); err != nil {
instance.Error = err
return
} else {
instance.SexOptions = make(map[string]gm.AttributeOption, len(sexOPtions))
for _, option := range sexOPtions {
instance.SexOptions[option.AdminName] = option
}
}
}()
//load sellers to memory
go func() {
defer instance.ImportWGroup.Done()
var vendors, err = gm.GetSellers(db)
if err != nil {
instance.Error = err
return
}
//init sellers map
instance.sellers = make(map[string]gm.MarketplaceSeller, len(vendors))
for _, vendor := range vendors {
instance.sellers[vendor.Url] = vendor
}
}()
if instance.Error != nil {
log.Println(instance.Error)
return nil, instance.Error
@ -237,7 +341,7 @@ func (importer *Importer) ImportProduct(product models.Product) (instance *Impor
}
if len(linkedProducts) > 1 {
//todo link products
var relation []gm.ProductRelation
for index, variant := range linkedProducts {
//spoint := "color" + strconv.Itoa(index)
@ -265,6 +369,13 @@ func (importer *Importer) ImportProduct(product models.Product) (instance *Impor
}
func (importer *Importer) importVariant(product models.Product) (*gm.Product, error) {
// check if wishlisted then update if.
//if _, ok := importer.wishlist[product.ProductNumber]; ok {
// delete(importer.wishlist,product.ProductNumber)
// return importer.updateVariant(product)
//}
productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
if categories, err := gm.GetCatKeywords(importer.baza, product.Categories); err != nil {
@ -285,14 +396,46 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er
tx := importer.baza.Begin()
if err := tx.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil {
tx.Rollback()
return nil, err
//todo update categories
byteErr, _ := json.Marshal(err)
var newError GormErr
if err1:= json.Unmarshal((byteErr), &newError);err1!=nil{
tx.Rollback()
log.Println(err1,"err2")
return nil, err1
}
if newError.Number == 1062{
var barProduct gm.Product
if err2 := tx.First(&barProduct,"sku = ?",mainPorduct.Sku).Error;err2 !=nil {
tx.Rollback()
log.Println(err2,"err3")
return nil, err2
}
if err3:= tx.Model(&barProduct).Association("Categories").Append(mainPorduct.Categories); err3!=nil{
tx.Rollback()
log.Println(err3,"err4")
return nil, err3
}
if err4 := tx.Commit().Error; err4 ==nil{
return importer.updateVariant(product)
}
}else{
tx.Rollback()
log.Println(err,"er1")
return nil, err
}
}
mainFlat := productRepo.makeProductFlat(mainPorduct.ID)
if err := tx.Create(&mainFlat).Error; err != nil {
tx.Rollback()
log.Println(err,"er5")
return nil, err
}
@ -304,9 +447,17 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er
}
savePoint := "size" + strconv.Itoa(index)
tx.SavePoint(savePoint)
sizeOPtion := gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue)
var sizeOPtion gm.AttributeOption
if variant.AttributeName == "Beden" {
sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue)
} else {
sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
}
sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku)
variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)
@ -330,19 +481,30 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er
if len(sizeVariants) == 0 {
tx.Rollback()
return nil, errors.New("siz variantlary yok bolsa main productam girayenok")
return nil, errors.New("size variantlary yok bolsa main productam girayenok")
} else {
calcPrice(sizeVariants, &mainFlat)
err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error
mainFlat.Variants = sizeVariants
if err != nil {
tx.Rollback()
log.Println(err,"er6")
return nil, err
}
}
}
sProduct := importer.createSellerProduct(&mainFlat, product.Vendor)
if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil {
tx.Rollback()
return nil, errSProduct
}
if err := tx.Commit().Error; err != nil {
return nil, err
}
@ -350,9 +512,43 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er
return &mainPorduct, nil
}
func (importer *Importer) createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct {
sellerID := importer.sellers[sellerURL].ID
if sellerID == 0 {
sellerID = 1
}
sellerProduct := gm.MarketplaceProduct{
MarketplaceSellerID: sellerID,
IsApproved: true,
Condition: "new",
Description: "scraped",
IsOwner: true,
ProductID: flat.ProductID,
}
for _, variant := range flat.Variants {
sellerProduct.Variants = append(sellerProduct.Variants, gm.MarketplaceProduct{
ProductID: variant.ProductID,
IsOwner: true,
IsApproved: true,
MarketplaceSellerID: sellerID,
Condition: "new",
})
}
return sellerProduct
}
func calcPrice(variants []gm.ProductFlat, flat *gm.ProductFlat) {
for _, variant := range variants {
if !variant.Status {
continue
}
if flat.MinPrice == 0 || flat.MinPrice > variant.MinPrice {
flat.MinPrice = variant.MinPrice
}
@ -394,3 +590,461 @@ func (importer *Importer) GetSexOption(optionName string) gm.AttributeOption {
importer.SexMutex.Unlock()
return option
}
func (importer *Importer) updateVariant(product models.Product) (*gm.Product, error) {
var flat gm.ProductFlat
err := importer.baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error
if err != nil {
if errors.Is(err,gorm.ErrRecordNotFound) {
return importer.importVariant(product)
}
//todo not found bolsa create etmeli
return nil, err
}
if flat.Product.Type == "configurable" {
productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
return nil, err
} else {
productRepo.Brand = brand
}
for _, variant := range *product.SizeVariants {
found := false
for _, flatVariant := range flat.Variants {
if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel {
if !variant.Sellable {
importer.baza.Model(&flatVariant).Update("status", false)
} else {
importer.updatePrice(variant.Price, &flatVariant)
if !flatVariant.Status {
importer.baza.Model(&flatVariant).Update("status", true)
}
}
found = true
break
}
}
if variant.Sellable && !found {
// insert variant
var sizeOPtion gm.AttributeOption
if variant.AttributeName == "Beden" {
sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["size"].ID, variant.AttributeValue)
} else {
sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
}
sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku)
variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)
if err := importer.baza.Omit("Categories.*").Create(&variantProduct).Error; err != nil {
log.Println("Variant Product Create Error: " + err.Error())
}else{
variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID)
if err := importer.baza.Create(&variantFlat).Error; err != nil {
log.Println("Variant Flat Create Error: " + err.Error())
}
flat.Variants = append(flat.Variants, variantFlat)
}
}
}
calcPrice(flat.Variants, &flat)
importer.baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat)
} else { //simple
importer.updatePrice(product.Price, &flat)
}
return &flat.Product, nil
}
func (importer *Importer) updatePrice(price models.Price, flat *gm.ProductFlat) {
if price.OriginalPrice.Value > price.DiscountedPrice.Value {
importer.baza.Model(&flat).Updates(map[string]interface{}{
"price": price.OriginalPrice.Value,
"special_price": price.DiscountedPrice.Value,
"min_price": price.DiscountedPrice.Value,
"max_price": price.OriginalPrice.Value,
})
importer.baza.Model(&gm.ProductAttributeValue{}).
Where("attribute_id = 11 and product_id = ?", flat.ProductID).
Update("float_value", price.OriginalPrice.Value)
importer.baza.Model(&gm.ProductAttributeValue{}).
Where("attribute_id = 13 and product_id = ?", flat.ProductID).
Update("float_value", price.DiscountedPrice.Value)
} else {
importer.baza.Model(&flat).Updates(map[string]interface{}{
"price": price.DiscountedPrice.Value,
"special_price": nil,
"min_price": price.DiscountedPrice.Value,
"max_price": price.DiscountedPrice.Value,
})
importer.baza.Model(&gm.ProductAttributeValue{}).
Where("attribute_id = 11 and product_id = ?", flat.ProductID).
Update("float_value", price.DiscountedPrice.Value)
importer.baza.Where("attribute_id = 13 and product_id = ?", flat.ProductID).
Delete(&gm.ProductAttributeValue{})
}
}
func (importer *Importer) UpdateOrCreate(product models.Product) (instance *Importer) {
firstProduct, err := importer.importVariant(product)
var newProducts []gm.Product
if err != nil {
helper.Error(err)
firstProduct, importer.Error = importer.updateVariant(product)
if importer.Error != nil {
return importer
}
} else if &firstProduct != nil {
newProducts = append(newProducts, *firstProduct)
}
if product.ColorVariants != nil && len(*product.ColorVariants) > 0 {
linkedProducts := []gm.Product{*firstProduct}
for _, colorVariant := range *product.ColorVariants {
var (
variant *gm.Product
err error
)
if !colorVariant.IsSellable {
if err = importer.baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil {
log.Println(err)
}
continue
} else {
if variant, err = importer.importVariant(colorVariant); err != nil {
if variant, importer.Error = importer.updateVariant(colorVariant); importer.Error != nil {
return importer
}
linkedProducts = append(linkedProducts, *variant)
} else {
newProducts = append(newProducts, *variant)
}
}
}
if len(newProducts) > 0 {
// relation
var relation []gm.ProductRelation
for _, linkedProduct := range linkedProducts {
for _, newProd := range newProducts {
relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID})
relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID})
}
}
if err := importer.baza.Create(&relation).Error; err != nil {
log.Println(err)
}
}
}
return importer
}
func (importer *Importer) importParsedVariant(product models.Product)(gm.Product, error) {
//todo search if exists
productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
productRepo.ImageType = "lcw"
if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
return gm.Product{}, err
} else {
productRepo.Brand = brand
}
mainPorduct := productRepo.makeProduct(importer)
//BEGIN TRANSACTION
tx := importer.baza.Begin()
if err := tx.Omit("Categories", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil {
tx.Rollback()
return gm.Product{}, err
}
mainFlat := productRepo.makeProductFlat(mainPorduct.ID)
if err := tx.Create(&mainFlat).Error; err != nil {
tx.Rollback()
return gm.Product{}, err
}
if productRepo.HasSizeVariants() {
var sizeVariants []gm.ProductFlat
for index, variant := range *product.SizeVariants {
if !variant.Sellable {
continue
}
savePoint := "size" + strconv.Itoa(index)
tx.SavePoint(savePoint)
var sizeOPtion gm.AttributeOption
if variant.AttributeName == "Beden" {
sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue)
} else {
sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
}
sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
log.Println(sku)
log.Println(variant)
variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku)
variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)
if err := tx.Omit("Categories.*").Create(&variantProduct).Error; err != nil {
log.Println("Variant Product Create Error: " + err.Error())
tx.RollbackTo(savePoint)
continue
}
variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, mainFlat.ID, variantProduct.ID)
if err := tx.Create(&variantFlat).Error; err != nil {
log.Println("Variant Flat Create Error: " + err.Error())
tx.RollbackTo(savePoint)
continue
}
sizeVariants = append(sizeVariants, variantFlat)
}
if len(sizeVariants) == 0 {
tx.Rollback()
return gm.Product{}, errors.New("siz variantlary yok bolsa main productam girayenok")
} else {
calcPrice(sizeVariants, &mainFlat)
err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error
mainFlat.Variants = sizeVariants
if err != nil {
tx.Rollback()
return gm.Product{}, err
}
}
}
log.Println(product.Vendor)
sProduct := importer.createSellerProduct(&mainFlat, product.Vendor)
if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil {
tx.Rollback()
return gm.Product{}, errSProduct
}
if err := tx.Commit().Error; err != nil {
return gm.Product{}, err
}
return mainPorduct, nil
}
func (importer *Importer) updateParsedVariant(product models.Product) (gm.Product, error){
var flat gm.ProductFlat
err := importer.baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error
if err != nil {
if errors.Is(err,gorm.ErrRecordNotFound) {
return importer.importParsedVariant(product)
}
//todo not found bolsa create etmeli
return gm.Product{}, err
}
if flat.Product.Type == "configurable" {
productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
productRepo.ImageType = "lcw"
if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
return gm.Product{}, err
} else {
productRepo.Brand = brand
}
for _, variant := range *product.SizeVariants {
found := false
for _, flatVariant := range flat.Variants {
if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel {
if !variant.Sellable {
importer.baza.Model(&flatVariant).Update("status", false)
} else {
importer.updatePrice(variant.Price, &flatVariant)
if !flatVariant.Status {
importer.baza.Model(&flatVariant).Update("status", true)
}
}
found = true
break
}
}
if variant.Sellable && !found {
// insert variant
var sizeOPtion gm.AttributeOption
if variant.AttributeName == "Beden" {
sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["size"].ID, variant.AttributeValue)
} else {
sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
}
log.Println(variant)
sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
log.Println(sku)
variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku)
variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)
if err := importer.baza.Omit("Categories").Create(&variantProduct).Error; err != nil {
log.Println("Variant Product Create Error: " + err.Error())
}
variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID)
if err := importer.baza.Create(&variantFlat).Error; err != nil {
log.Println("Variant Flat Create Error: " + err.Error())
}
flat.Variants = append(flat.Variants, variantFlat)
}
}
calcPrice(flat.Variants, &flat)
importer.baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat)
} else { //simple
importer.updatePrice(product.Price, &flat)
}
return flat.Product, nil
}
func (importer *Importer) UpdateOrCreateLCW(product models.Product) (instance *Importer){
var firstProduct gm.Product
result := importer.baza.First(&firstProduct,"sku=?",product.ProductNumber)
if errors.Is(result.Error, gorm.ErrRecordNotFound){
firstProduct, importer.Error = importer.importParsedVariant(product)
}else{
firstProduct, importer.Error = importer.updateParsedVariant(product)
}
var newProducts []gm.Product
if importer.Error != nil {
return importer
}else if &firstProduct != nil {
newProducts = append(newProducts, firstProduct)
}
if product.ColorVariants != nil && len(*product.ColorVariants) > 0 {
linkedProducts := []gm.Product{firstProduct}
for _, colorVariant := range *product.ColorVariants {
var (
variant gm.Product
err error
)
if !colorVariant.IsSellable {
if err = importer.baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil {
log.Println(err)
}
continue
} else {
//todo check parsed import variant
if variant, err = importer.importParsedVariant(colorVariant); err != nil {
if variant, importer.Error = importer.updateParsedVariant(colorVariant); importer.Error != nil {
return importer
}
linkedProducts = append(linkedProducts, variant)
} else {
newProducts = append(newProducts, variant)
}
}
}
if len(newProducts) > 0 {
// relation
var relation []gm.ProductRelation
for _, linkedProduct := range linkedProducts {
for _, newProd := range newProducts {
relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID})
relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID})
}
}
if err := importer.baza.Create(&relation).Error; err != nil {
log.Println(err)
}
}
}
return importer
}

View File

@ -0,0 +1,29 @@
package repositories
import (
"db_service/models"
helper "db_service/pkg"
)
type LinkParserLCW struct {
link string
}
func NewLinkParserLCW(link string) LinkParserLCW {
return LinkParserLCW{link: link}
}
func (l LinkParserLCW) ParseLink() (*models.Product, error) {
helper.Info("link: ", l.link)
lcwScraper := NewLCWScraper(l.link)
product, err := lcwScraper.InitProductDetailParsing()
if err != nil {
helper.Error(err)
return nil, err
}
return &product, nil
}

View File

@ -5,6 +5,7 @@ import (
helper "db_service/pkg"
"encoding/json"
"errors"
"fmt"
"net/http"
"reflect"
"regexp"
@ -76,8 +77,6 @@ func getProductIdFromShortLink(shortLink string) string {
productId = getProductIdFromLink(url)
helper.Info("productId: ", productId)
return productId
}
@ -117,6 +116,8 @@ func GetProductDetails(productId string) (models.TrendyolProductDetailModel, err
if err != nil {
helper.Error(err)
helper.Info(body)
return productDetailModel, err
}
@ -226,9 +227,9 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int
json["images"] = model.Images
json["brand"] = model.Brand.Name
json["cinsiyet"] = model.Gender.Name
json["description"] = model.Description
//json["description"] = model.Description
json["descriptions"] = model.ContentDescriptions
json["short_description"] = model.Description
//json["short_description"] = model.Description
// nested structure
json["price"] = make(map[string]interface{})
json["price"].(map[string]interface{})["originalPrice"] = model.Price.OriginalPrice
@ -266,7 +267,7 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int
var variants []models.Variant
// if show variants, then it is configurable product.
if model.ShowVariants {
if model.ShowVariants && len(model.Variants) != 0 {
for i := 0; i < len(model.Variants); i++ {
variant := model.Variants[i]
@ -288,30 +289,79 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int
variants = append(variants, variant)
}
}
}
}
}
for i := 0; i < len(model.AllVariants); i++ {
singleVariant := model.AllVariants[i]
// get the first variant for attribute info
fv := variants[0]
variant := models.Variant{
AttributeID: fv.AttributeID, //
AttributeName: fv.AttributeName, // Sample: "Beden"
AttributeType: fv.AttributeType, // Sample: "Size"
AttributeValue: singleVariant.Value,
Price: models.Price{
ProfitMargin: 0,
DiscountedPrice: struct {
Text string "json:\"text\""
Value float64 "json:\"value\""
}{
Text: fmt.Sprintf("%f", singleVariant.Price),
Value: singleVariant.Price,
},
SellingPrice: struct {
Text string "json:\"text\""
Value float64 "json:\"value\""
}{
Text: fmt.Sprintf("%f", singleVariant.Price),
Value: singleVariant.Price,
},
OriginalPrice: struct {
Text string "json:\"text\""
Value float64 "json:\"value\""
}{
Text: fmt.Sprintf("%f", singleVariant.Price),
Value: singleVariant.Price,
},
Currency: singleVariant.Currency,
},
ItemNumber: singleVariant.ItemNumber,
Sellable: singleVariant.InStock,
}
exists := helper.IsVariantsAdded(variants, variant.ItemNumber)
if !exists {
variants = append(variants, variant)
}
}
for i := 0; i < len(model.AlternativeVariants); i++ {
alternativeVariant := model.AlternativeVariants[i]
stockType := reflect.TypeOf(alternativeVariant.Stock)
if stockType == nil && len(variants) > 0 {
if len(variants) > 0 {
// get the first variant for attribute info
fv := variants[0]
variant := models.Variant{
AttributeID: fv.AttributeID,
AttributeName: fv.AttributeName,
AttributeType: fv.AttributeType,
AttributeID: fv.AttributeID, //
AttributeName: fv.AttributeName, // Sample: "Beden"
AttributeType: fv.AttributeType, // Sample: "Size"
AttributeValue: alternativeVariant.AttributeValue,
Price: alternativeVariant.Price,
ItemNumber: alternativeVariant.ItemNumber,
Sellable: alternativeVariant.Quantity > 0,
}
exists := helper.IsVariantsAdded(variants, variant.ItemNumber)
if !exists {
variants = append(variants, variant)
}
variants = append(variants, variant)
}
}

View File

@ -18,6 +18,7 @@ type ProductRepo struct {
Error error
ColorOption gm.AttributeOption
SexOption gm.AttributeOption
ImageType string
}
func InitProductRepo(data *models.Product, color, sex gm.AttributeOption) *ProductRepo {
@ -40,6 +41,7 @@ func InitProductRepo(data *models.Product, color, sex gm.AttributeOption) *Produ
Data: data,
ColorOption: color,
SexOption: sex,
ImageType: "cdn",
}
return instance
@ -87,7 +89,13 @@ func (pr *ProductRepo) makeProduct(imp *Importer) gm.Product {
if pr.HasSizeVariants() {
product.Type = "configurable"
product.SuperAttributes = []gm.Attribute{imp.AttributesMap["size"]}
if (*pr.Data.SizeVariants)[0].AttributeName == "Beden" {
product.SuperAttributes = []gm.Attribute{imp.AttributesMap["size"]}
} else {
product.SuperAttributes = []gm.Attribute{imp.AttributesMap["boyut"]}
product.AttributeFamilyID = 4 // todo fix hardcode
}
} else {
product.Type = "simple"
price := pr.Data.Price
@ -105,7 +113,7 @@ func (pr *ProductRepo) makeProduct(imp *Importer) gm.Product {
}
for _, element := range pr.Data.Images {
product.Images = append(product.Images, gm.ProductImage{Type: "cdn", Path: element})
product.Images = append(product.Images, gm.ProductImage{Type: pr.ImageType, Path: element})
}
return product
@ -156,6 +164,11 @@ func (pr *ProductRepo) makeProductFlat(productId uint) gm.ProductFlat {
flat.BrandID = pr.Brand.ID
}
if pr.Data.Cinsiyet != "" {
flat.Cinsiyet = int(pr.SexOption.ID)
flat.CinsiyetLabel = pr.Data.Cinsiyet
}
if !pr.HasSizeVariants() {
flat.MinPrice = pr.Data.Price.DiscountedPrice.Value
flat.MaxPrice = math.Max(pr.Data.Price.OriginalPrice.Value, pr.Data.Price.DiscountedPrice.Value)
@ -200,10 +213,18 @@ func (pr *ProductRepo) makeVariantFlat(variant models.Variant, SizID, parentID,
FavoritesCount: uint(pr.Data.FavoriteCount),
SizeLabel: variant.AttributeValue,
Size: int(SizID),
MaxPrice: maxPRice,
MinPrice: variant.Price.DiscountedPrice.Value,
Price: maxPRice,
ProductID: productID,
MaxPrice: maxPRice,
MinPrice: variant.Price.DiscountedPrice.Value,
Price: maxPRice,
ProductID: productID,
}
if variant.AttributeName == "Beden" {
flat.Size = int(SizID)
flat.SizeLabel = variant.AttributeValue
} else {
flat.Boyut = int(SizID)
flat.BoyutLabel = variant.AttributeValue
}
if flat.MaxPrice > flat.MinPrice {
@ -215,6 +236,15 @@ func (pr *ProductRepo) makeVariantFlat(variant models.Variant, SizID, parentID,
flat.ColorLabel = pr.Data.Color
}
if pr.Data.Brand != "" {
flat.BrandID = pr.Brand.ID
}
if pr.Data.Cinsiyet != "" {
flat.Cinsiyet = int(pr.SexOption.ID)
flat.CinsiyetLabel = pr.Data.Cinsiyet
}
return flat
}
@ -253,9 +283,14 @@ func (pr *ProductRepo) getVariantAttributes(AttributesMap map[string]gm.Attribut
{AttributeID: AttributesMap["name"].ID, TextValue: pr.Data.Name, Channel: "default", Locale: "tm"},
{AttributeID: AttributesMap["weight"].ID, TextValue: pr.Data.Weight},
{AttributeID: AttributesMap["status"].ID, BooleanValue: true},
{AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)},
//{AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)},
{AttributeID: AttributesMap["price"].ID, FloatValue: price},
}
if product.AttributeName == "Beden" {
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)})
} else {
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["boyut"].ID, IntegerValue: int(SizID)})
}
if pr.Data.Color != "" {
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["color"].ID, IntegerValue: int(pr.ColorOption.ID)})

301
repositories/scraper_lcw.go Normal file
View File

@ -0,0 +1,301 @@
package repositories
import (
"context"
"db_service/models"
helper "db_service/pkg"
"fmt"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/chromedp/cdproto/dom"
"github.com/chromedp/chromedp"
)
const (
LCW_URL = "https://www.lcwaikiki.com"
)
type LCWScraper struct {
link string
}
func NewLCWScraper(link string) LCWScraper {
return LCWScraper{link: link}
}
func (s LCWScraper) InitProductDetailParsing() (models.Product, error) {
product := models.Product{}
// load page with js
html, err := loadPage(s.link)
if err != nil {
return product, err
}
// convert to goquery document
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
if err != nil {
fmt.Println(err)
return product, err
}
product = parseDocument(doc, true)
product.Vendor = "lcw"
return product, nil
}
// loadPage loads page using chromeDp
// ChromeDp waits until JS render complete. returns html string
func loadPage(url string) (string, error) {
var res string
ctx, cancel := chromedp.NewContext(context.Background())
defer cancel()
err := chromedp.Run(ctx,
chromedp.Navigate(url),
chromedp.ActionFunc(func(ctx context.Context) error {
node, err := dom.GetDocument().Do(ctx)
if err != nil {
return err
}
res, err = dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx)
return err
}),
)
if err != nil {
return res, err
}
return res, nil
}
//parseDocument parses document using goQuery
func parseDocument(doc *goquery.Document, primaryModel bool) models.Product {
product := &models.Product{
ColorVariants: &([]models.Product{}),
SizeVariants: &([]models.Variant{}),
}
doc.Find("meta").Each(func(i int, s *goquery.Selection) {
name, _ := s.Attr("name")
content, _ := s.Attr("content")
switch name {
case "ModelId":
product.ProductGroupID = content
case "OptionId":
product.Sku = "p-" + content
product.ProductNumber = "lcw-"+content
case "ProductCode":
product.ProductCode = content
case "ProductName":
product.Name = content
case "BrandName":
product.Brand = content
case "Gender":
product.Cinsiyet = content
case "description":
product.Description = content
case "Color":
product.Color = content
case "DiscountPrice_1":
product.Price.SellingPrice.Text = ""
product.Price.SellingPrice.Value = convertStrToFloat(content)
case "CashPrice_1":
product.Price.OriginalPrice.Value = convertStrToFloat(content)
product.Weight = "0"
// case "Size":
// fmt.Println("SIZE: ", content)
// case "SizeId":
// fmt.Println("SIZE_ID: ", content)
}
})
productLink, ok := doc.Find(".share-link").Attr("value")
if ok {
urlKey := strings.ReplaceAll(productLink, LCW_URL, " ")
product.URLKey = strings.TrimSpace(urlKey)
}
setupPrices(product, doc)
setupSizeVariants(product, doc)
setupSizeImages(product, doc)
// call color variants for only primary model
if primaryModel {
setupColorVariants(product, doc)
}
setupProductProperties(product, doc)
return *product
}
func setupPrices(product *models.Product, doc *goquery.Document) {
basketPrice := strings.Trim(doc.Find(".col-xs-12 .price-area > .campaign-discount-detail > .basket-discount").Text(), " \n")
basketPriceFloat := convertStrToFloat(basketPrice)
// check basket price discount
if basketPriceFloat != 0.0 {
product.Price.DiscountedPrice.Value = basketPriceFloat
}
}
func setupSizeVariants(product *models.Product, doc *goquery.Document) {
optionSizes := doc.Find(".option-size").Children()
for i := range optionSizes.Nodes {
optionSizes.Eq(i).Each(func(ii int, selection *goquery.Selection) {
sizeValue, _ := selection.Attr("size")
dataStock, _ := selection.Attr("data-stock")
key, _ := selection.Attr("key")
keyInt, err := strconv.Atoi(key)
if err != nil {
// ... handle error
// panic(err)
fmt.Println("error in generating keyInt for ItemNumber")
}
if len(dataStock) != 0 {
stock, _ := strconv.Atoi(dataStock)
if stock > 3 {
sizeVariant := models.Variant{
AttributeName: "Beden",
AttributeValue: sizeValue,
Stock: dataStock,
Sellable: true,
ItemNumber: keyInt,
Price: models.Price{
DiscountedPrice: models.PriceValue{
Text: "",
Value: product.Price.DiscountedPrice.Value,
},
SellingPrice: models.PriceValue{
Text: "",
Value: product.Price.SellingPrice.Value,
},
OriginalPrice: models.PriceValue{
Text: "",
Value: product.Price.OriginalPrice.Value,
},
},
}
isItemAdded := helper.IsLCWSizeVariantsAdded(*product.SizeVariants, sizeVariant)
if !isItemAdded {
*product.SizeVariants = append(*product.SizeVariants, sizeVariant)
}
}
}
})
}
}
//setupColorVariants gets color variant products.
func setupColorVariants(product *models.Product, doc *goquery.Document) {
colorVariants := make([]models.Product, 0)
doc.Find(".color-option").Each(func(i int, s *goquery.Selection) {
optionLink, ok := s.Attr("href")
if ok && !strings.Contains(optionLink, "javascript:") {
url := LCW_URL + optionLink
htmlPage, err := loadPage(url)
if err != nil {
return
}
newDoc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlPage))
if err != nil {
fmt.Println("err: ", err)
return
}
colorOption := parseDocument(newDoc, false)
colorVariants = append(colorVariants, colorOption)
}
})
product.ColorVariantCount = len(colorVariants)
*product.ColorVariants = append(*product.ColorVariants, colorVariants...)
}
func setupSizeImages(product *models.Product, doc *goquery.Document) {
// images := make([]string, 0)
doc.Find("img").Each(func(i int, s *goquery.Selection) {
if image, ok := s.Attr("smallimages"); ok {
isAdded := helper.IsImageAdded(product.Images,image)
if !isAdded {
product.Images = append(product.Images, image)
}
}
})
}
func setupProductProperties(product *models.Product, doc *goquery.Document) {
// get collapseOne
collapseOne := doc.Find("#collapseOne")
// get descriptions
collapseOne.Find("li").Each(func(i int, selection *goquery.Selection) {
desc := strings.TrimSpace(selection.Text())
if len(desc) != 0 {
desc := models.DescriptionModel{
Description: desc,
Bold: false,
}
// descriptions = append(descriptions, desc)
product.Descriptions = append(product.Descriptions, desc)
}
})
collapseOne.Find(".option-info").Each(func(i int, selection *goquery.Selection) {
selection.Find("p").Each(func(i int, s *goquery.Selection) {
text := strings.Trim(s.Text(), "")
if len(text) != 0 {
attr := strings.Split(text, ":")
attrKey, attrValue := strings.Trim(attr[0], " "), strings.Trim(attr[1], " ")
if len(attrKey) != 0 && len(attrValue) != 0 {
attribute := map[string]string{
strings.ToLower(attrKey): attrValue,
}
product.Attributes = append(product.Attributes, attribute)
}
}
})
})
}
func convertStrToFloat(price string) float64 {
// remove TL
priceStr := strings.ReplaceAll(price, " TL", "")
// remove .(dot)
priceStr = strings.ReplaceAll(priceStr, ".", "")
// replace ,(comma) with .(dot)
priceStr = strings.ReplaceAll(priceStr, ",", ".")
// remove whitespace
priceStr = strings.ReplaceAll(priceStr, " ", "")
priceFloat, err := strconv.ParseFloat(priceStr, 64)
if err != nil {
return 0
}
return priceFloat
}

91
vendor/modules.txt vendored
View File

@ -1,6 +1,81 @@
# github.com/PuerkitoBio/goquery v1.8.0
## explicit; go 1.13
github.com/PuerkitoBio/goquery
# github.com/andybalholm/cascadia v1.3.1
## explicit; go 1.16
github.com/andybalholm/cascadia
# github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24
## explicit; go 1.19
github.com/chromedp/cdproto
github.com/chromedp/cdproto/accessibility
github.com/chromedp/cdproto/animation
github.com/chromedp/cdproto/audits
github.com/chromedp/cdproto/backgroundservice
github.com/chromedp/cdproto/browser
github.com/chromedp/cdproto/cachestorage
github.com/chromedp/cdproto/cast
github.com/chromedp/cdproto/cdp
github.com/chromedp/cdproto/css
github.com/chromedp/cdproto/database
github.com/chromedp/cdproto/debugger
github.com/chromedp/cdproto/deviceorientation
github.com/chromedp/cdproto/dom
github.com/chromedp/cdproto/domdebugger
github.com/chromedp/cdproto/domsnapshot
github.com/chromedp/cdproto/domstorage
github.com/chromedp/cdproto/emulation
github.com/chromedp/cdproto/eventbreakpoints
github.com/chromedp/cdproto/fetch
github.com/chromedp/cdproto/headlessexperimental
github.com/chromedp/cdproto/heapprofiler
github.com/chromedp/cdproto/indexeddb
github.com/chromedp/cdproto/input
github.com/chromedp/cdproto/inspector
github.com/chromedp/cdproto/io
github.com/chromedp/cdproto/layertree
github.com/chromedp/cdproto/log
github.com/chromedp/cdproto/media
github.com/chromedp/cdproto/memory
github.com/chromedp/cdproto/network
github.com/chromedp/cdproto/overlay
github.com/chromedp/cdproto/page
github.com/chromedp/cdproto/performance
github.com/chromedp/cdproto/performancetimeline
github.com/chromedp/cdproto/profiler
github.com/chromedp/cdproto/runtime
github.com/chromedp/cdproto/security
github.com/chromedp/cdproto/serviceworker
github.com/chromedp/cdproto/storage
github.com/chromedp/cdproto/systeminfo
github.com/chromedp/cdproto/target
github.com/chromedp/cdproto/tethering
github.com/chromedp/cdproto/tracing
github.com/chromedp/cdproto/webaudio
github.com/chromedp/cdproto/webauthn
# github.com/chromedp/chromedp v0.8.7
## explicit; go 1.18
github.com/chromedp/chromedp
github.com/chromedp/chromedp/device
github.com/chromedp/chromedp/kb
# github.com/chromedp/sysutil v1.0.0
## explicit; go 1.15
github.com/chromedp/sysutil
# github.com/go-sql-driver/mysql v1.6.0
## explicit; go 1.10
github.com/go-sql-driver/mysql
# github.com/gobwas/httphead v0.1.0
## explicit; go 1.15
github.com/gobwas/httphead
# github.com/gobwas/pool v0.2.1
## explicit
github.com/gobwas/pool
github.com/gobwas/pool/internal/pmath
github.com/gobwas/pool/pbufio
github.com/gobwas/pool/pbytes
# github.com/gobwas/ws v1.1.0
## explicit; go 1.15
github.com/gobwas/ws
github.com/gobwas/ws/wsutil
# github.com/gorilla/mux v1.8.0
## explicit; go 1.12
github.com/gorilla/mux
@ -19,9 +94,25 @@ github.com/jinzhu/now
# github.com/joho/godotenv v1.4.0
## explicit; go 1.12
github.com/joho/godotenv
# github.com/josharian/intern v1.0.0
## explicit; go 1.5
github.com/josharian/intern
# github.com/leesper/couchdb-golang v1.2.1
## explicit
github.com/leesper/couchdb-golang
# github.com/mailru/easyjson v0.7.7
## explicit; go 1.12
github.com/mailru/easyjson
github.com/mailru/easyjson/buffer
github.com/mailru/easyjson/jlexer
github.com/mailru/easyjson/jwriter
# golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
## explicit; go 1.17
golang.org/x/net/html
golang.org/x/net/html/atom
# golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec
## explicit; go 1.17
golang.org/x/sys/unix
# gorm.io/driver/mysql v1.3.5
## explicit; go 1.14
gorm.io/driver/mysql