diff --git a/controllers/importer.go b/controllers/ImportController.go similarity index 90% rename from controllers/importer.go rename to controllers/ImportController.go index 5a801ea..e03bbb1 100644 --- a/controllers/importer.go +++ b/controllers/ImportController.go @@ -1,4 +1,4 @@ -package importer +package controller import ( gm "db_service/gorm_models" @@ -18,14 +18,16 @@ import ( "gorm.io/gorm" ) -var mainCategories []gm.Category -var baza *gorm.DB -var mainImportWG, famAndSellerWG sync.WaitGroup -var families []gm.AttributeFamily -var sellers = make(map[string]gm.MarketplaceSeller) -var attributesMap = make(map[string]gm.Attribute) +var ( + mainCategories []gm.Category + baza *gorm.DB + mainImportWG, famAndSellerWG sync.WaitGroup + families []gm.AttributeFamily + sellers = make(map[string]gm.MarketplaceSeller) + attributesMap = make(map[string]gm.Attribute) +) -func Start(w http.ResponseWriter, route *http.Request) { +func StartImport(w http.ResponseWriter, route *http.Request) { start := time.Now() r := new(big.Int) @@ -95,7 +97,7 @@ func Start(w http.ResponseWriter, route *http.Request) { for _, element := range mainCategories { slug := element.Translations[0].Slug - go startImport("ty_db_"+slug, baza) + go importCategoryProducts("ty_db_"+slug, baza) // fmt.Println(<-result) } @@ -113,7 +115,7 @@ func Start(w http.ResponseWriter, route *http.Request) { } -func startImport(dbName string, db *gorm.DB) { +func importCategoryProducts(dbName string, db *gorm.DB) { defer mainImportWG.Done() dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName) @@ -135,7 +137,7 @@ func startImport(dbName string, db *gorm.DB) { skip += limit - body, err := helper.SendRequest("GET", url, nil, "", true) + body, err := helper.SendRequest("GET", url, nil, "") if err != nil { fmt.Println(err.Error()) @@ -151,7 +153,7 @@ func startImport(dbName string, db *gorm.DB) { //itearate 100 row products for _, element := range response.Rows { - importProduct(element.Doc, db) + ImportProduct(element.Doc, db) } } } else { @@ -166,7 +168,7 @@ func getTotalDocumentCount(db string) int { url := os.Getenv("couch_db_source") + db - body, err := helper.SendRequest("GET", url, nil, "", true) + body, err := helper.SendRequest("GET", url, nil, "") if err != nil { log.Println(err.Error()) @@ -206,7 +208,7 @@ func getCats(db *gorm.DB, catIDs []int) ([]gm.Category, string, error) { return categories, keywords, nil } -func importProduct(product models.Product, db *gorm.DB) { +func ImportProduct(product models.Product, db *gorm.DB) error { famAndSellerWG.Wait() //wait until attribute families and sellers are not get from mysql @@ -214,7 +216,7 @@ func importProduct(product models.Product, db *gorm.DB) { if errCat != nil { log.Println(errCat) - return + return errCat } var brand gm.Brand @@ -262,7 +264,7 @@ func importProduct(product models.Product, db *gorm.DB) { errMainProduct := db.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&iproduct).Error if errMainProduct != nil { log.Println(errMainProduct.Error()) - return + return errMainProduct } mainProductFlat.ProductID = iproduct.ID @@ -273,7 +275,7 @@ func importProduct(product models.Product, db *gorm.DB) { if errProductMainFlat != nil { log.Println(errProductMainFlat) - return + return errProductMainFlat } if len(product.ColorVariants) > 0 { @@ -381,7 +383,7 @@ func importProduct(product models.Product, db *gorm.DB) { if errProdVariant != nil { log.Println(errProdVariant) - return + return errProdVariant } flatVariant.ProductID = productVariant.ID @@ -390,7 +392,7 @@ func importProduct(product models.Product, db *gorm.DB) { if errVariant != nil { log.Println(errVariant) - return + return errVariant } mainProductFlat.Variants = append(mainProductFlat.Variants, flatVariant) @@ -431,7 +433,7 @@ func importProduct(product models.Product, db *gorm.DB) { if errProdVariant != nil { log.Println(errProdVariant) - return + return errProdVariant } variantFlat.ProductID = productVariant.ID @@ -444,7 +446,7 @@ func importProduct(product models.Product, db *gorm.DB) { if errVariant != nil { log.Println(errVariant) - return + return errVariant } mainProductFlat.Variants = append(mainProductFlat.Variants, variantFlat) @@ -540,7 +542,7 @@ func importProduct(product models.Product, db *gorm.DB) { if errSizeVar != nil { log.Println(errSizeVar) - return + return errSizeVar } flatVariant.ProductID = sizeVariantProduct.ID @@ -549,7 +551,7 @@ func importProduct(product models.Product, db *gorm.DB) { if errVariant != nil { log.Println(errVariant) - return + return errVariant } mainProductFlat.Variants = append(mainProductFlat.Variants, flatVariant) @@ -571,6 +573,8 @@ func importProduct(product models.Product, db *gorm.DB) { if errSProduct != nil { log.Println(errSProduct) } + + return nil } func createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct { @@ -719,3 +723,36 @@ func prepearAttributesWithFlat(data *models.Product) ([]gm.ProductAttributeValue return productAttributeValues, flat } + +//func productAttributesAndFlat(data *models.Product) ([]gm.ProductAttributeValue,gm.ProductFlat){ +// +// var description string +// +// for _, desc := range data.Descriptions { +// description += "

" + desc.Description + "

" +// } +// +// weight, _ := strconv.ParseFloat(data.Weight, 64) +// +// flat := gm.ProductFlat{ +// Status: true, +// VisibleIndividually: true, +// Name: data.Name, +// Sku: data.ProductGroupID, +// ProductNumber: data.ProductNumber, +// Description: description, +// UrlKey: data.ProductGroupID, +// Weight: weight, +// FavoritesCount: uint(data.FavoriteCount), +// } +// return []gm.ProductAttributeValue{ +// {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount}, +// {AttributeID: attributesMap["source"].ID, TextValue: data.URLKey}, +// {AttributeID: attributesMap["product_number"].ID, TextValue: data.ProductNumber}, +// {AttributeID: attributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"}, +// {AttributeID: attributesMap["weight"].ID, TextValue: data.Weight}, +// {AttributeID: attributesMap["status"].ID, BooleanValue: true}, +// {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true}, +// {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, +// },flat +//} diff --git a/controllers/ParseController.go b/controllers/ParseController.go new file mode 100644 index 0000000..5aafcdd --- /dev/null +++ b/controllers/ParseController.go @@ -0,0 +1,64 @@ +package controller + +import ( + "db_service/gorm_models" + "db_service/models" + helper "db_service/pkg" + "db_service/repositories" + "encoding/json" + "fmt" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "log" + "net/http" + "os" + "strconv" + "time" +) + +func ParseLink(w http.ResponseWriter, route *http.Request) { + start := time.Now() + + link := route.URL.Query().Get("url") + helper.Info("link: ", link) + + linkParser := repositories.NewLinkParser(link) + + //todo + product, err := linkParser.ParseLink() + + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(map[string]string{ + "msg": err.Error(), + }) + + return + } + + jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID) + + if err != nil { + helper.Error(err) + return + } + + baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{}) + + if err != nil { + helper.Error(err) + return + } + + ImportProduct(jsonProduct, baza) + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]string{ + "msg": "Link parsed successfully", + "productGroupId": strconv.Itoa(product.ProductGroupID), + }) + + elapsed := time.Since(start) + log.Printf("end parse took %s", elapsed) + http.Error(w, fmt.Sprintf("end import took %s", elapsed), http.StatusOK) +} diff --git a/go.mod b/go.mod index 9b0a8ed..ca0a144 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gosimple/slug v1.12.0 github.com/joho/godotenv v1.4.0 + github.com/leesper/couchdb-golang v1.2.1 gorm.io/driver/mysql v1.3.5 gorm.io/gorm v1.23.8 ) diff --git a/go.sum b/go.sum index 323b508..18b7db5 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/leesper/couchdb-golang v1.2.1 h1:FqSaTxxT2mVRLbxGQVkZakRRoSzWhPmV8UEKYjA/GWc= +github.com/leesper/couchdb-golang v1.2.1/go.mod h1:OU3FDAM3mazHx15oi8Hm+egTMneBUqepwnh0LuBSH54= gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY= gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE= diff --git a/main.go b/main.go index 28bc039..b0001ce 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ package main import ( - importer "db_service/controllers" + "db_service/controllers" "github.com/gorilla/mux" "github.com/joho/godotenv" "log" @@ -23,17 +23,12 @@ func init() { func main() { route := mux.NewRouter() - route.HandleFunc("/init-importer", importer.Start) + route.HandleFunc("/init-importer", controller.StartImport) + route.HandleFunc("/parse-link", controller.ParseLink) err := http.ListenAndServe(os.Getenv("port"), route) if err != nil { log.Fatal("error: ", err) } - //err := importer.Start() - //if err != nil { - // fmt.Println("err: ", err.Error()) - //} - - // dene() } diff --git a/models/category.go b/models/category.go index c43afff..3238a79 100644 --- a/models/category.go +++ b/models/category.go @@ -16,3 +16,15 @@ type Category struct { type BagistoResponse struct { Data []Category } +type CouchCategory struct { + Rev string `json:"_rev"` + CreatedAt string `json:"createdAt"` + ID string `json:"id"` + Name string `json:"name"` + Order string `json:"order"` + ParentID string `json:"parent_id"` + SargaID string `json:"sarga_id"` + Slug string `json:"slug"` + UpdatedAt string `json:"updatedAt"` + Weight string `json:"weight"` +} diff --git a/models/product.go b/models/product.go index b4f7a54..5d2d9f6 100644 --- a/models/product.go +++ b/models/product.go @@ -77,3 +77,237 @@ type Row struct { type BagistoModelResponse struct { Rows []Row `json:"rows"` } + +type TrendyolProductDetailResponse struct { + IsSuccess bool `json:"isSuccess"` + StatusCode int `json:"statusCode"` + Error interface{} `json:"error"` + Result TrendyolProductDetailModel `json:"result"` + Headers struct { + Tysidecarcachable string `json:"tysidecarcachable"` + } `json:"headers"` +} + +type TrendyolProductDetailModel struct { + AlternativeVariants []AlternativeVariant `json:"alternativeVariants"` + Attributes []struct { + Key struct { + Name string `json:"name"` + ID int `json:"id"` + } `json:"key"` + Value struct { + Name string `json:"name"` + ID int `json:"id"` + } `json:"value"` + Starred bool `json:"starred"` + } `json:"attributes"` + Variants []Variant `json:"variants"` + OtherMerchants []interface{} `json:"otherMerchants"` + Campaign struct { + ID int `json:"id"` + Name string `json:"name"` + StartDate string `json:"startDate"` + EndDate string `json:"endDate"` + IsMultipleSupplied bool `json:"isMultipleSupplied"` + StockTypeID int `json:"stockTypeId"` + URL string `json:"url"` + ShowTimer bool `json:"showTimer"` + } `json:"campaign"` + Category struct { + ID int `json:"id"` + Name string `json:"name"` + Hierarchy string `json:"hierarchy"` + Refundable bool `json:"refundable"` + BeautifiedName string `json:"beautifiedName"` + IsVASEnabled bool `json:"isVASEnabled"` + } `json:"category"` + Brand struct { + IsVirtual bool `json:"isVirtual"` + BeautifiedName string `json:"beautifiedName"` + ID int `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + } `json:"brand"` + Color string `json:"color"` + MetaBrand struct { + ID int `json:"id"` + Name string `json:"name"` + BeautifiedName string `json:"beautifiedName"` + IsVirtual bool `json:"isVirtual"` + Path string `json:"path"` + } `json:"metaBrand"` + ShowVariants bool `json:"showVariants"` + ShowSexualContent bool `json:"showSexualContent"` + BrandCategoryBanners []interface{} `json:"brandCategoryBanners"` + AllVariants []struct { + ItemNumber int `json:"itemNumber"` + Value string `json:"value"` + InStock bool `json:"inStock"` + Currency string `json:"currency"` + Barcode string `json:"barcode"` + Price float64 `json:"price"` + } `json:"allVariants"` + OtherMerchantVariants []interface{} `json:"otherMerchantVariants"` + InstallmentBanner interface{} `json:"installmentBanner"` + IsVasEnabled bool `json:"isVasEnabled"` + OriginalCategory struct { + ID int `json:"id"` + Name string `json:"name"` + Hierarchy string `json:"hierarchy"` + Refundable bool `json:"refundable"` + BeautifiedName string `json:"beautifiedName"` + IsVASEnabled bool `json:"isVASEnabled"` + } `json:"originalCategory"` + Landings []interface{} `json:"landings"` + ID int `json:"id"` + ProductCode string `json:"productCode"` + Name string `json:"name"` + NameWithProductCode string `json:"nameWithProductCode"` + Description string `json:"description"` + ContentDescriptions []struct { + Description string `json:"description"` + Bold bool `json:"bold"` + } `json:"contentDescriptions"` + ProductGroupID int `json:"productGroupId"` + Tax int `json:"tax"` + BusinessUnit string `json:"businessUnit"` + MaxInstallment int `json:"maxInstallment"` + Gender struct { + Name string `json:"name"` + ID int `json:"id"` + } `json:"gender"` + URL string `json:"url"` + Images []string `json:"images"` + IsSellable bool `json:"isSellable"` + IsBasketDiscount bool `json:"isBasketDiscount"` + HasStock bool `json:"hasStock"` + Price Price `json:"price"` + IsFreeCargo bool `json:"isFreeCargo"` + Promotions []struct { + PromotionRemainingTime string `json:"promotionRemainingTime"` + Type int `json:"type"` + Text string `json:"text"` + ID int `json:"id"` + Link string `json:"link"` + } `json:"promotions"` + Merchant struct { + IsSearchableMerchant bool `json:"isSearchableMerchant"` + Stickers []interface{} `json:"stickers"` + ID int `json:"id"` + Name string `json:"name"` + OfficialName string `json:"officialName"` + CityName string `json:"cityName"` + TaxNumber string `json:"taxNumber"` + SellerScore float64 `json:"sellerScore"` + SellerScoreColor string `json:"sellerScoreColor"` + DeliveryProviderName string `json:"deliveryProviderName"` + SellerLink string `json:"sellerLink"` + } `json:"merchant"` + DeliveryInformation struct { + IsRushDelivery bool `json:"isRushDelivery"` + DeliveryDate string `json:"deliveryDate"` + } `json:"deliveryInformation"` + CargoRemainingDays int `json:"cargoRemainingDays"` + IsMarketplace bool `json:"isMarketplace"` + ProductStamps []struct { + Type string `json:"type"` + ImageURL string `json:"imageUrl"` + Position string `json:"position"` + AspectRatio float64 `json:"aspectRatio"` + Priority int `json:"priority"` + PriceTagStamp bool `json:"priceTagStamp,omitempty"` + } `json:"productStamps"` + HasHTMLContent bool `json:"hasHtmlContent"` + FavoriteCount int `json:"favoriteCount"` + UxLayout string `json:"uxLayout"` + IsDigitalGood bool `json:"isDigitalGood"` + IsRunningOut bool `json:"isRunningOut"` + ScheduledDelivery bool `json:"scheduledDelivery"` + RatingScore struct { + AverageRating float64 `json:"averageRating"` + TotalRatingCount int `json:"totalRatingCount"` + TotalCommentCount int `json:"totalCommentCount"` + } `json:"ratingScore"` + ShowStarredAttributes bool `json:"showStarredAttributes"` + ReviewsURL string `json:"reviewsUrl"` + QuestionsURL string `json:"questionsUrl"` + SellerQuestionEnabled bool `json:"sellerQuestionEnabled"` + SizeExpectationAvailable bool `json:"sizeExpectationAvailable"` + CrossPromotionAward struct { + AwardType interface{} `json:"awardType"` + AwardValue interface{} `json:"awardValue"` + ContentID int `json:"contentId"` + MerchantID int `json:"merchantId"` + } `json:"crossPromotionAward"` + RushDeliveryMerchantListingExist bool `json:"rushDeliveryMerchantListingExist"` + LowerPriceMerchantListingExist bool `json:"lowerPriceMerchantListingExist"` + ShowValidFlashSales bool `json:"showValidFlashSales"` + ShowExpiredFlashSales bool `json:"showExpiredFlashSales"` + WalletRebate struct { + MinPrice int `json:"minPrice"` + MaxPrice int `json:"maxPrice"` + RebateRatio float64 `json:"rebateRatio"` + } `json:"walletRebate"` + IsArtWork bool `json:"isArtWork"` +} + +type TrendyolProductVariantsResponse struct { + IsSuccess bool `json:"isSuccess"` + StatusCode int `json:"statusCode"` + Error interface{} `json:"error"` + Result struct { + SlicingAttributes []struct { + Brand struct { + BeautifiedName string `json:"beautifiedName"` + ID int `json:"id"` + Name string `json:"name"` + IsVirtual bool `json:"isVirtual"` + Path string `json:"path"` + } `json:"brand"` + Attributes []struct { + Contents []struct { + URL string `json:"url"` + ID int `json:"id"` + ImageURL string `json:"imageUrl"` + Name string `json:"name"` + Price struct { + DiscountedPrice struct { + Text string `json:"text"` + Value float64 `json:"value"` + } `json:"discountedPrice"` + OriginalPrice struct { + Text string `json:"text"` + Value float64 `json:"value"` + } `json:"originalPrice"` + SellingPrice struct { + Text string `json:"text"` + Value float64 `json:"value"` + } `json:"sellingPrice"` + } `json:"price"` + } `json:"contents"` + Name string `json:"name"` + BeautifiedName string `json:"beautifiedName"` + } `json:"attributes"` + Type string `json:"type"` + DisplayName string `json:"displayName"` + Order int `json:"order"` + DisplayType int `json:"displayType"` + } `json:"slicingAttributes"` + } `json:"result"` + Headers struct { + Tysidecarcachable string `json:"tysidecarcachable"` + } `json:"headers"` +} +type AlternativeVariant struct { + AttributeValue string `json:"attributeValue"` + AttributeBeautifiedValue string `json:"attributeBeautifiedValue"` + CampaignID int `json:"campaignId"` + MerchantID int `json:"merchantId"` + URLQuery string `json:"urlQuery"` + ListingID string `json:"listingId"` + ItemNumber int `json:"itemNumber"` + Barcode string `json:"barcode"` + Stock interface{} `json:"stock"` + Quantity int `json:"quantity"` + Price Price `json:"price"` +} diff --git a/pkg/global.go b/pkg/global.go new file mode 100644 index 0000000..49bd491 --- /dev/null +++ b/pkg/global.go @@ -0,0 +1,12 @@ +package helper + +import ( + "db_service/models" + "github.com/leesper/couchdb-golang" +) + +// CdbServer is global couchdb server +var CdbServer *couchdb.Server + +// Categories stores all categories and their weight +var Categories []models.CouchCategory = []models.CouchCategory{} diff --git a/pkg/helper.go b/pkg/helper.go index 7c1db22..85a67f2 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -5,6 +5,7 @@ import ( "log" "net" "net/http" + "os" "time" ) @@ -64,3 +65,22 @@ func CheckDBExists(endpoint string) bool { return false } } + +func GetCategoryWeight(slug string) string { + + db, err := CdbServer.Get(os.Getenv("db_ty_categories")) + + if err != nil { + Error("can not get db. Err: " + err.Error()) + return "0.5" + } + + doc, err := db.Get(slug, nil) + + if err != nil { + Error("can not get by slug. Err: " + err.Error()) + return "0.5" + } + + return doc["weight"].(string) +} diff --git a/pkg/log.go b/pkg/log.go new file mode 100644 index 0000000..9b039d4 --- /dev/null +++ b/pkg/log.go @@ -0,0 +1,86 @@ +package helper + +import ( + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "time" +) + +type Level int + +var ( + DefaultPrefix = "" + DefaultCallerDepth = 2 + + logger *log.Logger + logPrefix = "" + levelFlags = []string{"DEBUG", "INFO", "WARN", "ERROR", "FATAL"} +) + +const ( + DEBUG Level = iota + INFO + WARNING + ERROR + FATAL +) + +// Setup initialize the log instance +func Setup() { + logger = log.New(os.Stdout, DefaultPrefix, log.Flags()&^(log.Ldate|log.Ltime)) +} + +// Debug output logs at debug level +func Debug(v ...interface{}) { + setPrefix(DEBUG) + logger.Println(v...) +} + +// Info output logs at info level +func Info(v ...interface{}) { + setPrefix(INFO) + logger.Println(v...) +} + +// Warn output logs at warn level +func Warn(v ...interface{}) { + setPrefix(WARNING) + logger.Println(v...) +} + +// Error output logs at error level +func Error(v ...interface{}) { + prefix := setPrefix(ERROR) + logger.Println(v...) + + prefix.msg = v +} + +// Fatal output logs at fatal level +func Fatal(v ...interface{}) { + setPrefix(FATAL) + logger.Fatalln(v...) +} + +// setPrefix set the prefix of the log output +func setPrefix(level Level) prefix { + _, file, line, ok := runtime.Caller(DefaultCallerDepth) + if ok { + logPrefix = fmt.Sprintf("[%s:%d] [%s] ", filepath.Base(file), line, levelFlags[level]) + } else { + logPrefix = fmt.Sprintf("[%s] ", levelFlags[level]) + } + + logPrefix += "| " + time.Now().Format("01.02.2006 15:04:05") + " | " + + logger.SetPrefix(logPrefix) + return prefix{prefix: logPrefix} +} + +type prefix struct { + prefix string + msg interface{} +} diff --git a/pkg/request.go b/pkg/request.go index 871e7b2..b11e60d 100644 --- a/pkg/request.go +++ b/pkg/request.go @@ -5,16 +5,15 @@ import ( "context" "fmt" "io/ioutil" - "log" "net" "net/http" "time" ) -func SendRequest(method string, endpoint string, values []byte, authKey string, isCouchDbReq bool) ([]byte, error) { +func SendRequest(method string, endpoint string, values []byte, authKey string) ([]byte, error) { const ConnectMaxWaitTime = 30 * time.Second - const RequestMaxWaitTime = 10 * time.Minute + const RequestMaxWaitTime = 60 * time.Second client := http.Client{ Transport: &http.Transport{ @@ -32,7 +31,7 @@ func SendRequest(method string, endpoint string, values []byte, authKey string, req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewBuffer(values)) req.Proto = "HTTP/2.0" if err != nil { - log.Println(err.Error()) + Error(err) return emptyBody, err } @@ -49,45 +48,27 @@ func SendRequest(method string, endpoint string, values []byte, authKey string, if response != nil { response.Body.Close() } - log.Println(err.Error()) + Error(err) return emptyBody, err } else if err != nil { if response != nil { response.Body.Close() } - log.Println(err.Error()) + Error(err) return emptyBody, err } - - // 200 OK - // 201 Created - // 202 Accepted - // 404 Not Found - // 409 Conflict - - if isCouchDbReq { - // fmt.Printf("responseStatusCode: %d\n", response.StatusCode) - if response.StatusCode == http.StatusNotFound { - if response != nil { - response.Body.Close() - } - return emptyBody, nil - } - // TODO: handle conflict error - } else { - if response.StatusCode != http.StatusOK { - if response != nil { - response.Body.Close() - } - err := fmt.Errorf("response: code: %d, body: %v", response.StatusCode, response.Body) - log.Println(err.Error()) - return emptyBody, err + if response.StatusCode != http.StatusOK { + if response != nil { + response.Body.Close() } + err := fmt.Errorf("response: code: %d, body: %v", response.StatusCode, response.Body) + Error(err) + return emptyBody, err } body, err := ioutil.ReadAll(response.Body) if err != nil { - log.Println(err.Error()) + Error(err) if response != nil { response.Body.Close() } @@ -99,3 +80,20 @@ func SendRequest(method string, endpoint string, values []byte, authKey string, return body, nil } +func NewHttpClient() (http.Client, context.Context) { + const ConnectMaxWaitTime = 80 * time.Second + const RequestMaxWaitTime = 120 * time.Second + + client := http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: ConnectMaxWaitTime, + }).DialContext, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), RequestMaxWaitTime) + defer cancel() + + return client, ctx +} diff --git a/repositories/ParseRepository.go b/repositories/ParseRepository.go new file mode 100644 index 0000000..dd0e372 --- /dev/null +++ b/repositories/ParseRepository.go @@ -0,0 +1,325 @@ +package repositories + +import ( + "db_service/models" + helper "db_service/pkg" + "encoding/json" + "errors" + "net/http" + "reflect" + "regexp" + "strconv" + "strings" +) + +type LinkParser struct { + link string +} + +func NewLinkParser(link string) LinkParser { + return LinkParser{link: link} +} + +func (l LinkParser) ParseLink() (models.TrendyolProductDetailModel, error) { + + helper.Info("link: ", l.link) + + productId := "" + + if isShortLink(l.link) { + productId = getProductIdFromShortLink(l.link) + } else { + productId = getProductIdFromLink(l.link) + } + + if len(productId) == 0 { + parseErr := errors.New("can not parse product id") + helper.Error(parseErr) + return models.TrendyolProductDetailModel{}, parseErr + } + + helper.Info("productId: ", productId) + + return GetProductDetails(productId) +} + +func getProductIdFromShortLink(shortLink string) string { + + var productId string + + client, _ := helper.NewHttpClient() + req, err := http.NewRequest("GET", shortLink, nil) + req.Proto = "HTTP/2.0" + if err != nil { + helper.Error(err) + return "" + } + + q := req.URL.Query() + req.URL.RawQuery = q.Encode() + req.Header.Set("Connection", "close") + req = req.WithContext(req.Context()) + req.Close = true + client.CloseIdleConnections() + + response, err := client.Do(req) + if err != nil { + helper.Error(err) + return productId + } + + defer response.Body.Close() + + url := response.Request.URL.Path + + helper.Info("link url: ", url) + + productId = getProductIdFromLink(url) + + helper.Info("productId: ", productId) + + return productId +} + +func getProductIdFromLink(link string) string { + var productId string + + if strings.Contains(link, "?") { + link = strings.Split(link, "?")[0] + } + + strArr := strings.Split(link, "-") + productId = strArr[len(strArr)-1] + + return productId +} + +func isShortLink(link string) bool { + return !strings.Contains(link, "trendyol.com") +} + +// GetProductDetails return JSON object. Merges color option +func GetProductDetails(productId string) (models.TrendyolProductDetailModel, error) { + + var response models.TrendyolProductDetailResponse + productDetailModel := models.TrendyolProductDetailModel{} + + // set linearVariants false to get variants + url := "https://public.trendyol.com/discovery-web-productgw-service/api/productDetail/" + productId + "?storefrontId=1&culture=tr-TR&linearVariants=false" + + body, err := helper.SendRequest("GET", url, nil, "") + + if err != nil { + return productDetailModel, err + } + + err = json.Unmarshal(body, &response) + + if err != nil { + helper.Error(err) + return productDetailModel, err + } + + productDetailModel = response.Result + + return productDetailModel, nil +} + +// getProductDetailWithOptions returns JSON with variants +func (l LinkParser) GetProductDetailWithOptions(productId, productGroupId int) (models.Product, error) { + + primaryProductDetail, err := GetProductDetails(strconv.Itoa(productId)) + + if err != nil { + return models.Product{}, err + } + + productDetailJSON := CreateJSONFromModel(primaryProductDetail) + + if productDetailJSON != nil { + colorVariants, err := GetProductColorVariants(productGroupId) + + var colorVariantsJson []map[string]interface{} + + if err != nil { + return models.Product{}, err + } + + // get the color variant products + for _, slicingAttribute := range colorVariants.Result.SlicingAttributes { + for _, attribute := range slicingAttribute.Attributes { + for _, content := range attribute.Contents { + // Don't fetch primary product again. + if content.ID != primaryProductDetail.ID { + productVariantDetail, errGPD := GetProductDetails(strconv.Itoa(content.ID)) + if errGPD == nil { + productVariantJSON := CreateJSONFromModel(productVariantDetail) + if productVariantJSON != nil { + colorVariantsJson = append(colorVariantsJson, productVariantJSON) + } + } + } + } + } + // Color variant count + if len(slicingAttribute.Attributes) > 0 { + productDetailJSON["color_variant_count"] = len(slicingAttribute.Attributes) + productDetailJSON["color_variants"] = colorVariantsJson + } + } + } + + jsonString, _ := json.Marshal(productDetailJSON) + + // convert json to struct + converted := models.Product{} + json.Unmarshal(jsonString, &converted) + + return converted, nil +} + +// getProductColorVariants returns color options of product +func GetProductColorVariants(productGroupId int) (models.TrendyolProductVariantsResponse, error) { + + url := "https://public.trendyol.com/discovery-web-productgw-service/api/productGroup/" + strconv.Itoa(productGroupId) + "?storefrontId=1&culture=tr-TR" + + var response models.TrendyolProductVariantsResponse + + body, err := helper.SendRequest("GET", url, nil, "") + + if err != nil { + return response, err + } + + err = json.Unmarshal(body, &response) + + if err != nil { + helper.Error(err) + return response, err + } + + return response, nil +} + +// createJSONFromModel creates json from [trendyol.TrendyolProductDetailModel] and returns +func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]interface{} { + + json := map[string]interface{}{} + + // get weight from categories (Stored in helper array (RAM)) + weight := helper.GetCategoryWeight(model.Category.BeautifiedName) + productGroupId := strconv.Itoa(model.ProductGroupID) + + json["_id"] = productGroupId + json["product_group_id"] = productGroupId + json["vendor"] = "trendyol" + json["sku"] = "p-" + strconv.Itoa(model.ID) + json["product_number"] = strconv.Itoa(model.ID) + json["product_code"] = model.ProductCode + json["name"] = model.Name + json["sellable"] = model.IsSellable + json["favorite_count"] = model.FavoriteCount + json["weight"] = weight + json["name_with_product_code"] = model.NameWithProductCode + json["url_key"] = "https://www.trendyol.com" + model.URL + json["images"] = model.Images + json["brand"] = model.Brand.Name + json["cinsiyet"] = model.Gender.Name + json["description"] = model.Description + json["descriptions"] = model.ContentDescriptions + json["short_description"] = model.Description + // nested structure + json["price"] = make(map[string]interface{}) + json["price"].(map[string]interface{})["originalPrice"] = model.Price.OriginalPrice + json["price"].(map[string]interface{})["sellingPrice"] = model.Price.SellingPrice + json["price"].(map[string]interface{})["discountedPrice"] = model.Price.DiscountedPrice + + attrLen := len(model.Attributes) + + if attrLen != 0 { + for i := 0; i < attrLen; i++ { + attribute := model.Attributes[i] + if attribute.Key.Name == "Renk" { + json["color"] = attribute.Value.Name + } + } + } + + // set categories with value 1 + json["categories"] = []int{1} + + attributes := make([]map[string]string, 0) + for _, attr := range model.Attributes { + var re = regexp.MustCompile(`/[^A-Z0-9]/ig`) + keyStr := re.ReplaceAllString(attr.Key.Name, `_`) + key := strings.ToLower(keyStr) + attribute := map[string]string{ + key: attr.Value.Name, + } + + attributes = append(attributes, attribute) + } + + json["attributes"] = attributes + + var variants []models.Variant + + // if show variants, then it is configurable product. + if model.ShowVariants { + + for i := 0; i < len(model.Variants); i++ { + variant := model.Variants[i] + + if variant.Sellable { + stockType := reflect.TypeOf(variant.Stock) + + if stockType == nil { + variants = append(variants, variant) + } else { + // scrape is via link parse + // we need to parse other variants + + // convert stock to int + stock, ok := variant.Stock.(float64) + + if ok { + if stock > 0 { + variants = append(variants, variant) + } + } + + } + } + + } + + for i := 0; i < len(model.AlternativeVariants); i++ { + alternativeVariant := model.AlternativeVariants[i] + + stockType := reflect.TypeOf(alternativeVariant.Stock) + + if stockType == nil && len(variants) > 0 { + + // get the first variant for attribute info + fv := variants[0] + + variant := models.Variant{ + AttributeID: fv.AttributeID, + AttributeName: fv.AttributeName, + AttributeType: fv.AttributeType, + AttributeValue: alternativeVariant.AttributeValue, + Price: alternativeVariant.Price, + } + variants = append(variants, variant) + } + } + + json["size_variants"] = variants + } + + if model.ShowVariants && len(variants) == 0 { + return nil + } + + return json +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 63fac51..0a2dac3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -19,6 +19,9 @@ github.com/jinzhu/now # github.com/joho/godotenv v1.4.0 ## explicit; go 1.12 github.com/joho/godotenv +# github.com/leesper/couchdb-golang v1.2.1 +## explicit +github.com/leesper/couchdb-golang # gorm.io/driver/mysql v1.3.5 ## explicit; go 1.14 gorm.io/driver/mysql