link parse test ready
This commit is contained in:
parent
62617e2b6b
commit
841c4c66b3
|
|
@ -1,4 +1,4 @@
|
|||
package importer
|
||||
package controller
|
||||
|
||||
import (
|
||||
gm "db_service/gorm_models"
|
||||
|
|
@ -18,14 +18,16 @@ import (
|
|||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var mainCategories []gm.Category
|
||||
var baza *gorm.DB
|
||||
var mainImportWG, famAndSellerWG sync.WaitGroup
|
||||
var families []gm.AttributeFamily
|
||||
var sellers = make(map[string]gm.MarketplaceSeller)
|
||||
var attributesMap = make(map[string]gm.Attribute)
|
||||
var (
|
||||
mainCategories []gm.Category
|
||||
baza *gorm.DB
|
||||
mainImportWG, famAndSellerWG sync.WaitGroup
|
||||
families []gm.AttributeFamily
|
||||
sellers = make(map[string]gm.MarketplaceSeller)
|
||||
attributesMap = make(map[string]gm.Attribute)
|
||||
)
|
||||
|
||||
func Start(w http.ResponseWriter, route *http.Request) {
|
||||
func StartImport(w http.ResponseWriter, route *http.Request) {
|
||||
start := time.Now()
|
||||
|
||||
r := new(big.Int)
|
||||
|
|
@ -95,7 +97,7 @@ func Start(w http.ResponseWriter, route *http.Request) {
|
|||
for _, element := range mainCategories {
|
||||
slug := element.Translations[0].Slug
|
||||
|
||||
go startImport("ty_db_"+slug, baza)
|
||||
go importCategoryProducts("ty_db_"+slug, baza)
|
||||
|
||||
// fmt.Println(<-result)
|
||||
}
|
||||
|
|
@ -113,7 +115,7 @@ func Start(w http.ResponseWriter, route *http.Request) {
|
|||
|
||||
}
|
||||
|
||||
func startImport(dbName string, db *gorm.DB) {
|
||||
func importCategoryProducts(dbName string, db *gorm.DB) {
|
||||
defer mainImportWG.Done()
|
||||
|
||||
dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName)
|
||||
|
|
@ -135,7 +137,7 @@ func startImport(dbName string, db *gorm.DB) {
|
|||
|
||||
skip += limit
|
||||
|
||||
body, err := helper.SendRequest("GET", url, nil, "", true)
|
||||
body, err := helper.SendRequest("GET", url, nil, "")
|
||||
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
|
|
@ -151,7 +153,7 @@ func startImport(dbName string, db *gorm.DB) {
|
|||
|
||||
//itearate 100 row products
|
||||
for _, element := range response.Rows {
|
||||
importProduct(element.Doc, db)
|
||||
ImportProduct(element.Doc, db)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
@ -166,7 +168,7 @@ func getTotalDocumentCount(db string) int {
|
|||
|
||||
url := os.Getenv("couch_db_source") + db
|
||||
|
||||
body, err := helper.SendRequest("GET", url, nil, "", true)
|
||||
body, err := helper.SendRequest("GET", url, nil, "")
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
|
|
@ -206,7 +208,7 @@ func getCats(db *gorm.DB, catIDs []int) ([]gm.Category, string, error) {
|
|||
return categories, keywords, nil
|
||||
}
|
||||
|
||||
func importProduct(product models.Product, db *gorm.DB) {
|
||||
func ImportProduct(product models.Product, db *gorm.DB) error {
|
||||
|
||||
famAndSellerWG.Wait() //wait until attribute families and sellers are not get from mysql
|
||||
|
||||
|
|
@ -214,7 +216,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
|
||||
if errCat != nil {
|
||||
log.Println(errCat)
|
||||
return
|
||||
return errCat
|
||||
}
|
||||
|
||||
var brand gm.Brand
|
||||
|
|
@ -262,7 +264,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
errMainProduct := db.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&iproduct).Error
|
||||
if errMainProduct != nil {
|
||||
log.Println(errMainProduct.Error())
|
||||
return
|
||||
return errMainProduct
|
||||
}
|
||||
|
||||
mainProductFlat.ProductID = iproduct.ID
|
||||
|
|
@ -273,7 +275,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
|
||||
if errProductMainFlat != nil {
|
||||
log.Println(errProductMainFlat)
|
||||
return
|
||||
return errProductMainFlat
|
||||
}
|
||||
|
||||
if len(product.ColorVariants) > 0 {
|
||||
|
|
@ -381,7 +383,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
|
||||
if errProdVariant != nil {
|
||||
log.Println(errProdVariant)
|
||||
return
|
||||
return errProdVariant
|
||||
}
|
||||
|
||||
flatVariant.ProductID = productVariant.ID
|
||||
|
|
@ -390,7 +392,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
|
||||
if errVariant != nil {
|
||||
log.Println(errVariant)
|
||||
return
|
||||
return errVariant
|
||||
}
|
||||
|
||||
mainProductFlat.Variants = append(mainProductFlat.Variants, flatVariant)
|
||||
|
|
@ -431,7 +433,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
|
||||
if errProdVariant != nil {
|
||||
log.Println(errProdVariant)
|
||||
return
|
||||
return errProdVariant
|
||||
}
|
||||
|
||||
variantFlat.ProductID = productVariant.ID
|
||||
|
|
@ -444,7 +446,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
|
||||
if errVariant != nil {
|
||||
log.Println(errVariant)
|
||||
return
|
||||
return errVariant
|
||||
}
|
||||
|
||||
mainProductFlat.Variants = append(mainProductFlat.Variants, variantFlat)
|
||||
|
|
@ -540,7 +542,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
|
||||
if errSizeVar != nil {
|
||||
log.Println(errSizeVar)
|
||||
return
|
||||
return errSizeVar
|
||||
}
|
||||
|
||||
flatVariant.ProductID = sizeVariantProduct.ID
|
||||
|
|
@ -549,7 +551,7 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
|
||||
if errVariant != nil {
|
||||
log.Println(errVariant)
|
||||
return
|
||||
return errVariant
|
||||
}
|
||||
|
||||
mainProductFlat.Variants = append(mainProductFlat.Variants, flatVariant)
|
||||
|
|
@ -571,6 +573,8 @@ func importProduct(product models.Product, db *gorm.DB) {
|
|||
if errSProduct != nil {
|
||||
log.Println(errSProduct)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct {
|
||||
|
|
@ -719,3 +723,36 @@ func prepearAttributesWithFlat(data *models.Product) ([]gm.ProductAttributeValue
|
|||
|
||||
return productAttributeValues, flat
|
||||
}
|
||||
|
||||
//func productAttributesAndFlat(data *models.Product) ([]gm.ProductAttributeValue,gm.ProductFlat){
|
||||
//
|
||||
// var description string
|
||||
//
|
||||
// for _, desc := range data.Descriptions {
|
||||
// description += "<p>" + desc.Description + "</p>"
|
||||
// }
|
||||
//
|
||||
// weight, _ := strconv.ParseFloat(data.Weight, 64)
|
||||
//
|
||||
// flat := gm.ProductFlat{
|
||||
// Status: true,
|
||||
// VisibleIndividually: true,
|
||||
// Name: data.Name,
|
||||
// Sku: data.ProductGroupID,
|
||||
// ProductNumber: data.ProductNumber,
|
||||
// Description: description,
|
||||
// UrlKey: data.ProductGroupID,
|
||||
// Weight: weight,
|
||||
// FavoritesCount: uint(data.FavoriteCount),
|
||||
// }
|
||||
// return []gm.ProductAttributeValue{
|
||||
// {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount},
|
||||
// {AttributeID: attributesMap["source"].ID, TextValue: data.URLKey},
|
||||
// {AttributeID: attributesMap["product_number"].ID, TextValue: data.ProductNumber},
|
||||
// {AttributeID: attributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"},
|
||||
// {AttributeID: attributesMap["weight"].ID, TextValue: data.Weight},
|
||||
// {AttributeID: attributesMap["status"].ID, BooleanValue: true},
|
||||
// {AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true},
|
||||
// {AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
|
||||
// },flat
|
||||
//}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
package controller
|
||||
|
||||
import (
|
||||
"db_service/gorm_models"
|
||||
"db_service/models"
|
||||
helper "db_service/pkg"
|
||||
"db_service/repositories"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
func ParseLink(w http.ResponseWriter, route *http.Request) {
|
||||
start := time.Now()
|
||||
|
||||
link := route.URL.Query().Get("url")
|
||||
helper.Info("link: ", link)
|
||||
|
||||
linkParser := repositories.NewLinkParser(link)
|
||||
|
||||
//todo
|
||||
product, err := linkParser.ParseLink()
|
||||
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
json.NewEncoder(w).Encode(map[string]string{
|
||||
"msg": err.Error(),
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID)
|
||||
|
||||
if err != nil {
|
||||
helper.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{})
|
||||
|
||||
if err != nil {
|
||||
helper.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
ImportProduct(jsonProduct, baza)
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]string{
|
||||
"msg": "Link parsed successfully",
|
||||
"productGroupId": strconv.Itoa(product.ProductGroupID),
|
||||
})
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("end parse took %s", elapsed)
|
||||
http.Error(w, fmt.Sprintf("end import took %s", elapsed), http.StatusOK)
|
||||
}
|
||||
1
go.mod
1
go.mod
|
|
@ -6,6 +6,7 @@ require (
|
|||
github.com/gorilla/mux v1.8.0
|
||||
github.com/gosimple/slug v1.12.0
|
||||
github.com/joho/godotenv v1.4.0
|
||||
github.com/leesper/couchdb-golang v1.2.1
|
||||
gorm.io/driver/mysql v1.3.5
|
||||
gorm.io/gorm v1.23.8
|
||||
)
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -13,6 +13,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
|||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/leesper/couchdb-golang v1.2.1 h1:FqSaTxxT2mVRLbxGQVkZakRRoSzWhPmV8UEKYjA/GWc=
|
||||
github.com/leesper/couchdb-golang v1.2.1/go.mod h1:OU3FDAM3mazHx15oi8Hm+egTMneBUqepwnh0LuBSH54=
|
||||
gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY=
|
||||
gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
|
||||
gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=
|
||||
|
|
|
|||
11
main.go
11
main.go
|
|
@ -1,7 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
importer "db_service/controllers"
|
||||
"db_service/controllers"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/joho/godotenv"
|
||||
"log"
|
||||
|
|
@ -23,17 +23,12 @@ func init() {
|
|||
func main() {
|
||||
|
||||
route := mux.NewRouter()
|
||||
route.HandleFunc("/init-importer", importer.Start)
|
||||
route.HandleFunc("/init-importer", controller.StartImport)
|
||||
route.HandleFunc("/parse-link", controller.ParseLink)
|
||||
err := http.ListenAndServe(os.Getenv("port"), route)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal("error: ", err)
|
||||
}
|
||||
//err := importer.Start()
|
||||
//if err != nil {
|
||||
// fmt.Println("err: ", err.Error())
|
||||
//}
|
||||
|
||||
// dene()
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,3 +16,15 @@ type Category struct {
|
|||
type BagistoResponse struct {
|
||||
Data []Category
|
||||
}
|
||||
type CouchCategory struct {
|
||||
Rev string `json:"_rev"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Order string `json:"order"`
|
||||
ParentID string `json:"parent_id"`
|
||||
SargaID string `json:"sarga_id"`
|
||||
Slug string `json:"slug"`
|
||||
UpdatedAt string `json:"updatedAt"`
|
||||
Weight string `json:"weight"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,3 +77,237 @@ type Row struct {
|
|||
type BagistoModelResponse struct {
|
||||
Rows []Row `json:"rows"`
|
||||
}
|
||||
|
||||
type TrendyolProductDetailResponse struct {
|
||||
IsSuccess bool `json:"isSuccess"`
|
||||
StatusCode int `json:"statusCode"`
|
||||
Error interface{} `json:"error"`
|
||||
Result TrendyolProductDetailModel `json:"result"`
|
||||
Headers struct {
|
||||
Tysidecarcachable string `json:"tysidecarcachable"`
|
||||
} `json:"headers"`
|
||||
}
|
||||
|
||||
type TrendyolProductDetailModel struct {
|
||||
AlternativeVariants []AlternativeVariant `json:"alternativeVariants"`
|
||||
Attributes []struct {
|
||||
Key struct {
|
||||
Name string `json:"name"`
|
||||
ID int `json:"id"`
|
||||
} `json:"key"`
|
||||
Value struct {
|
||||
Name string `json:"name"`
|
||||
ID int `json:"id"`
|
||||
} `json:"value"`
|
||||
Starred bool `json:"starred"`
|
||||
} `json:"attributes"`
|
||||
Variants []Variant `json:"variants"`
|
||||
OtherMerchants []interface{} `json:"otherMerchants"`
|
||||
Campaign struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
StartDate string `json:"startDate"`
|
||||
EndDate string `json:"endDate"`
|
||||
IsMultipleSupplied bool `json:"isMultipleSupplied"`
|
||||
StockTypeID int `json:"stockTypeId"`
|
||||
URL string `json:"url"`
|
||||
ShowTimer bool `json:"showTimer"`
|
||||
} `json:"campaign"`
|
||||
Category struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Hierarchy string `json:"hierarchy"`
|
||||
Refundable bool `json:"refundable"`
|
||||
BeautifiedName string `json:"beautifiedName"`
|
||||
IsVASEnabled bool `json:"isVASEnabled"`
|
||||
} `json:"category"`
|
||||
Brand struct {
|
||||
IsVirtual bool `json:"isVirtual"`
|
||||
BeautifiedName string `json:"beautifiedName"`
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
} `json:"brand"`
|
||||
Color string `json:"color"`
|
||||
MetaBrand struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
BeautifiedName string `json:"beautifiedName"`
|
||||
IsVirtual bool `json:"isVirtual"`
|
||||
Path string `json:"path"`
|
||||
} `json:"metaBrand"`
|
||||
ShowVariants bool `json:"showVariants"`
|
||||
ShowSexualContent bool `json:"showSexualContent"`
|
||||
BrandCategoryBanners []interface{} `json:"brandCategoryBanners"`
|
||||
AllVariants []struct {
|
||||
ItemNumber int `json:"itemNumber"`
|
||||
Value string `json:"value"`
|
||||
InStock bool `json:"inStock"`
|
||||
Currency string `json:"currency"`
|
||||
Barcode string `json:"barcode"`
|
||||
Price float64 `json:"price"`
|
||||
} `json:"allVariants"`
|
||||
OtherMerchantVariants []interface{} `json:"otherMerchantVariants"`
|
||||
InstallmentBanner interface{} `json:"installmentBanner"`
|
||||
IsVasEnabled bool `json:"isVasEnabled"`
|
||||
OriginalCategory struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Hierarchy string `json:"hierarchy"`
|
||||
Refundable bool `json:"refundable"`
|
||||
BeautifiedName string `json:"beautifiedName"`
|
||||
IsVASEnabled bool `json:"isVASEnabled"`
|
||||
} `json:"originalCategory"`
|
||||
Landings []interface{} `json:"landings"`
|
||||
ID int `json:"id"`
|
||||
ProductCode string `json:"productCode"`
|
||||
Name string `json:"name"`
|
||||
NameWithProductCode string `json:"nameWithProductCode"`
|
||||
Description string `json:"description"`
|
||||
ContentDescriptions []struct {
|
||||
Description string `json:"description"`
|
||||
Bold bool `json:"bold"`
|
||||
} `json:"contentDescriptions"`
|
||||
ProductGroupID int `json:"productGroupId"`
|
||||
Tax int `json:"tax"`
|
||||
BusinessUnit string `json:"businessUnit"`
|
||||
MaxInstallment int `json:"maxInstallment"`
|
||||
Gender struct {
|
||||
Name string `json:"name"`
|
||||
ID int `json:"id"`
|
||||
} `json:"gender"`
|
||||
URL string `json:"url"`
|
||||
Images []string `json:"images"`
|
||||
IsSellable bool `json:"isSellable"`
|
||||
IsBasketDiscount bool `json:"isBasketDiscount"`
|
||||
HasStock bool `json:"hasStock"`
|
||||
Price Price `json:"price"`
|
||||
IsFreeCargo bool `json:"isFreeCargo"`
|
||||
Promotions []struct {
|
||||
PromotionRemainingTime string `json:"promotionRemainingTime"`
|
||||
Type int `json:"type"`
|
||||
Text string `json:"text"`
|
||||
ID int `json:"id"`
|
||||
Link string `json:"link"`
|
||||
} `json:"promotions"`
|
||||
Merchant struct {
|
||||
IsSearchableMerchant bool `json:"isSearchableMerchant"`
|
||||
Stickers []interface{} `json:"stickers"`
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
OfficialName string `json:"officialName"`
|
||||
CityName string `json:"cityName"`
|
||||
TaxNumber string `json:"taxNumber"`
|
||||
SellerScore float64 `json:"sellerScore"`
|
||||
SellerScoreColor string `json:"sellerScoreColor"`
|
||||
DeliveryProviderName string `json:"deliveryProviderName"`
|
||||
SellerLink string `json:"sellerLink"`
|
||||
} `json:"merchant"`
|
||||
DeliveryInformation struct {
|
||||
IsRushDelivery bool `json:"isRushDelivery"`
|
||||
DeliveryDate string `json:"deliveryDate"`
|
||||
} `json:"deliveryInformation"`
|
||||
CargoRemainingDays int `json:"cargoRemainingDays"`
|
||||
IsMarketplace bool `json:"isMarketplace"`
|
||||
ProductStamps []struct {
|
||||
Type string `json:"type"`
|
||||
ImageURL string `json:"imageUrl"`
|
||||
Position string `json:"position"`
|
||||
AspectRatio float64 `json:"aspectRatio"`
|
||||
Priority int `json:"priority"`
|
||||
PriceTagStamp bool `json:"priceTagStamp,omitempty"`
|
||||
} `json:"productStamps"`
|
||||
HasHTMLContent bool `json:"hasHtmlContent"`
|
||||
FavoriteCount int `json:"favoriteCount"`
|
||||
UxLayout string `json:"uxLayout"`
|
||||
IsDigitalGood bool `json:"isDigitalGood"`
|
||||
IsRunningOut bool `json:"isRunningOut"`
|
||||
ScheduledDelivery bool `json:"scheduledDelivery"`
|
||||
RatingScore struct {
|
||||
AverageRating float64 `json:"averageRating"`
|
||||
TotalRatingCount int `json:"totalRatingCount"`
|
||||
TotalCommentCount int `json:"totalCommentCount"`
|
||||
} `json:"ratingScore"`
|
||||
ShowStarredAttributes bool `json:"showStarredAttributes"`
|
||||
ReviewsURL string `json:"reviewsUrl"`
|
||||
QuestionsURL string `json:"questionsUrl"`
|
||||
SellerQuestionEnabled bool `json:"sellerQuestionEnabled"`
|
||||
SizeExpectationAvailable bool `json:"sizeExpectationAvailable"`
|
||||
CrossPromotionAward struct {
|
||||
AwardType interface{} `json:"awardType"`
|
||||
AwardValue interface{} `json:"awardValue"`
|
||||
ContentID int `json:"contentId"`
|
||||
MerchantID int `json:"merchantId"`
|
||||
} `json:"crossPromotionAward"`
|
||||
RushDeliveryMerchantListingExist bool `json:"rushDeliveryMerchantListingExist"`
|
||||
LowerPriceMerchantListingExist bool `json:"lowerPriceMerchantListingExist"`
|
||||
ShowValidFlashSales bool `json:"showValidFlashSales"`
|
||||
ShowExpiredFlashSales bool `json:"showExpiredFlashSales"`
|
||||
WalletRebate struct {
|
||||
MinPrice int `json:"minPrice"`
|
||||
MaxPrice int `json:"maxPrice"`
|
||||
RebateRatio float64 `json:"rebateRatio"`
|
||||
} `json:"walletRebate"`
|
||||
IsArtWork bool `json:"isArtWork"`
|
||||
}
|
||||
|
||||
type TrendyolProductVariantsResponse struct {
|
||||
IsSuccess bool `json:"isSuccess"`
|
||||
StatusCode int `json:"statusCode"`
|
||||
Error interface{} `json:"error"`
|
||||
Result struct {
|
||||
SlicingAttributes []struct {
|
||||
Brand struct {
|
||||
BeautifiedName string `json:"beautifiedName"`
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
IsVirtual bool `json:"isVirtual"`
|
||||
Path string `json:"path"`
|
||||
} `json:"brand"`
|
||||
Attributes []struct {
|
||||
Contents []struct {
|
||||
URL string `json:"url"`
|
||||
ID int `json:"id"`
|
||||
ImageURL string `json:"imageUrl"`
|
||||
Name string `json:"name"`
|
||||
Price struct {
|
||||
DiscountedPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"discountedPrice"`
|
||||
OriginalPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"originalPrice"`
|
||||
SellingPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"sellingPrice"`
|
||||
} `json:"price"`
|
||||
} `json:"contents"`
|
||||
Name string `json:"name"`
|
||||
BeautifiedName string `json:"beautifiedName"`
|
||||
} `json:"attributes"`
|
||||
Type string `json:"type"`
|
||||
DisplayName string `json:"displayName"`
|
||||
Order int `json:"order"`
|
||||
DisplayType int `json:"displayType"`
|
||||
} `json:"slicingAttributes"`
|
||||
} `json:"result"`
|
||||
Headers struct {
|
||||
Tysidecarcachable string `json:"tysidecarcachable"`
|
||||
} `json:"headers"`
|
||||
}
|
||||
type AlternativeVariant struct {
|
||||
AttributeValue string `json:"attributeValue"`
|
||||
AttributeBeautifiedValue string `json:"attributeBeautifiedValue"`
|
||||
CampaignID int `json:"campaignId"`
|
||||
MerchantID int `json:"merchantId"`
|
||||
URLQuery string `json:"urlQuery"`
|
||||
ListingID string `json:"listingId"`
|
||||
ItemNumber int `json:"itemNumber"`
|
||||
Barcode string `json:"barcode"`
|
||||
Stock interface{} `json:"stock"`
|
||||
Quantity int `json:"quantity"`
|
||||
Price Price `json:"price"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
package helper
|
||||
|
||||
import (
|
||||
"db_service/models"
|
||||
"github.com/leesper/couchdb-golang"
|
||||
)
|
||||
|
||||
// CdbServer is global couchdb server
|
||||
var CdbServer *couchdb.Server
|
||||
|
||||
// Categories stores all categories and their weight
|
||||
var Categories []models.CouchCategory = []models.CouchCategory{}
|
||||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -64,3 +65,22 @@ func CheckDBExists(endpoint string) bool {
|
|||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func GetCategoryWeight(slug string) string {
|
||||
|
||||
db, err := CdbServer.Get(os.Getenv("db_ty_categories"))
|
||||
|
||||
if err != nil {
|
||||
Error("can not get db. Err: " + err.Error())
|
||||
return "0.5"
|
||||
}
|
||||
|
||||
doc, err := db.Get(slug, nil)
|
||||
|
||||
if err != nil {
|
||||
Error("can not get by slug. Err: " + err.Error())
|
||||
return "0.5"
|
||||
}
|
||||
|
||||
return doc["weight"].(string)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
package helper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Level int
|
||||
|
||||
var (
|
||||
DefaultPrefix = ""
|
||||
DefaultCallerDepth = 2
|
||||
|
||||
logger *log.Logger
|
||||
logPrefix = ""
|
||||
levelFlags = []string{"DEBUG", "INFO", "WARN", "ERROR", "FATAL"}
|
||||
)
|
||||
|
||||
const (
|
||||
DEBUG Level = iota
|
||||
INFO
|
||||
WARNING
|
||||
ERROR
|
||||
FATAL
|
||||
)
|
||||
|
||||
// Setup initialize the log instance
|
||||
func Setup() {
|
||||
logger = log.New(os.Stdout, DefaultPrefix, log.Flags()&^(log.Ldate|log.Ltime))
|
||||
}
|
||||
|
||||
// Debug output logs at debug level
|
||||
func Debug(v ...interface{}) {
|
||||
setPrefix(DEBUG)
|
||||
logger.Println(v...)
|
||||
}
|
||||
|
||||
// Info output logs at info level
|
||||
func Info(v ...interface{}) {
|
||||
setPrefix(INFO)
|
||||
logger.Println(v...)
|
||||
}
|
||||
|
||||
// Warn output logs at warn level
|
||||
func Warn(v ...interface{}) {
|
||||
setPrefix(WARNING)
|
||||
logger.Println(v...)
|
||||
}
|
||||
|
||||
// Error output logs at error level
|
||||
func Error(v ...interface{}) {
|
||||
prefix := setPrefix(ERROR)
|
||||
logger.Println(v...)
|
||||
|
||||
prefix.msg = v
|
||||
}
|
||||
|
||||
// Fatal output logs at fatal level
|
||||
func Fatal(v ...interface{}) {
|
||||
setPrefix(FATAL)
|
||||
logger.Fatalln(v...)
|
||||
}
|
||||
|
||||
// setPrefix set the prefix of the log output
|
||||
func setPrefix(level Level) prefix {
|
||||
_, file, line, ok := runtime.Caller(DefaultCallerDepth)
|
||||
if ok {
|
||||
logPrefix = fmt.Sprintf("[%s:%d] [%s] ", filepath.Base(file), line, levelFlags[level])
|
||||
} else {
|
||||
logPrefix = fmt.Sprintf("[%s] ", levelFlags[level])
|
||||
}
|
||||
|
||||
logPrefix += "| " + time.Now().Format("01.02.2006 15:04:05") + " | "
|
||||
|
||||
logger.SetPrefix(logPrefix)
|
||||
return prefix{prefix: logPrefix}
|
||||
}
|
||||
|
||||
type prefix struct {
|
||||
prefix string
|
||||
msg interface{}
|
||||
}
|
||||
|
|
@ -5,16 +5,15 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
func SendRequest(method string, endpoint string, values []byte, authKey string, isCouchDbReq bool) ([]byte, error) {
|
||||
func SendRequest(method string, endpoint string, values []byte, authKey string) ([]byte, error) {
|
||||
|
||||
const ConnectMaxWaitTime = 30 * time.Second
|
||||
const RequestMaxWaitTime = 10 * time.Minute
|
||||
const RequestMaxWaitTime = 60 * time.Second
|
||||
|
||||
client := http.Client{
|
||||
Transport: &http.Transport{
|
||||
|
|
@ -32,7 +31,7 @@ func SendRequest(method string, endpoint string, values []byte, authKey string,
|
|||
req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewBuffer(values))
|
||||
req.Proto = "HTTP/2.0"
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
Error(err)
|
||||
return emptyBody, err
|
||||
}
|
||||
|
||||
|
|
@ -49,45 +48,27 @@ func SendRequest(method string, endpoint string, values []byte, authKey string,
|
|||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
log.Println(err.Error())
|
||||
Error(err)
|
||||
return emptyBody, err
|
||||
} else if err != nil {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
log.Println(err.Error())
|
||||
Error(err)
|
||||
return emptyBody, err
|
||||
}
|
||||
|
||||
// 200 OK
|
||||
// 201 Created
|
||||
// 202 Accepted
|
||||
// 404 Not Found
|
||||
// 409 Conflict
|
||||
|
||||
if isCouchDbReq {
|
||||
// fmt.Printf("responseStatusCode: %d\n", response.StatusCode)
|
||||
if response.StatusCode == http.StatusNotFound {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
return emptyBody, nil
|
||||
}
|
||||
// TODO: handle conflict error
|
||||
} else {
|
||||
if response.StatusCode != http.StatusOK {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
err := fmt.Errorf("response: code: %d, body: %v", response.StatusCode, response.Body)
|
||||
log.Println(err.Error())
|
||||
return emptyBody, err
|
||||
if response.StatusCode != http.StatusOK {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
err := fmt.Errorf("response: code: %d, body: %v", response.StatusCode, response.Body)
|
||||
Error(err)
|
||||
return emptyBody, err
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
Error(err)
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
|
|
@ -99,3 +80,20 @@ func SendRequest(method string, endpoint string, values []byte, authKey string,
|
|||
|
||||
return body, nil
|
||||
}
|
||||
func NewHttpClient() (http.Client, context.Context) {
|
||||
const ConnectMaxWaitTime = 80 * time.Second
|
||||
const RequestMaxWaitTime = 120 * time.Second
|
||||
|
||||
client := http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: ConnectMaxWaitTime,
|
||||
}).DialContext,
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), RequestMaxWaitTime)
|
||||
defer cancel()
|
||||
|
||||
return client, ctx
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,325 @@
|
|||
package repositories
|
||||
|
||||
import (
|
||||
"db_service/models"
|
||||
helper "db_service/pkg"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type LinkParser struct {
|
||||
link string
|
||||
}
|
||||
|
||||
func NewLinkParser(link string) LinkParser {
|
||||
return LinkParser{link: link}
|
||||
}
|
||||
|
||||
func (l LinkParser) ParseLink() (models.TrendyolProductDetailModel, error) {
|
||||
|
||||
helper.Info("link: ", l.link)
|
||||
|
||||
productId := ""
|
||||
|
||||
if isShortLink(l.link) {
|
||||
productId = getProductIdFromShortLink(l.link)
|
||||
} else {
|
||||
productId = getProductIdFromLink(l.link)
|
||||
}
|
||||
|
||||
if len(productId) == 0 {
|
||||
parseErr := errors.New("can not parse product id")
|
||||
helper.Error(parseErr)
|
||||
return models.TrendyolProductDetailModel{}, parseErr
|
||||
}
|
||||
|
||||
helper.Info("productId: ", productId)
|
||||
|
||||
return GetProductDetails(productId)
|
||||
}
|
||||
|
||||
func getProductIdFromShortLink(shortLink string) string {
|
||||
|
||||
var productId string
|
||||
|
||||
client, _ := helper.NewHttpClient()
|
||||
req, err := http.NewRequest("GET", shortLink, nil)
|
||||
req.Proto = "HTTP/2.0"
|
||||
if err != nil {
|
||||
helper.Error(err)
|
||||
return ""
|
||||
}
|
||||
|
||||
q := req.URL.Query()
|
||||
req.URL.RawQuery = q.Encode()
|
||||
req.Header.Set("Connection", "close")
|
||||
req = req.WithContext(req.Context())
|
||||
req.Close = true
|
||||
client.CloseIdleConnections()
|
||||
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
helper.Error(err)
|
||||
return productId
|
||||
}
|
||||
|
||||
defer response.Body.Close()
|
||||
|
||||
url := response.Request.URL.Path
|
||||
|
||||
helper.Info("link url: ", url)
|
||||
|
||||
productId = getProductIdFromLink(url)
|
||||
|
||||
helper.Info("productId: ", productId)
|
||||
|
||||
return productId
|
||||
}
|
||||
|
||||
func getProductIdFromLink(link string) string {
|
||||
var productId string
|
||||
|
||||
if strings.Contains(link, "?") {
|
||||
link = strings.Split(link, "?")[0]
|
||||
}
|
||||
|
||||
strArr := strings.Split(link, "-")
|
||||
productId = strArr[len(strArr)-1]
|
||||
|
||||
return productId
|
||||
}
|
||||
|
||||
func isShortLink(link string) bool {
|
||||
return !strings.Contains(link, "trendyol.com")
|
||||
}
|
||||
|
||||
// GetProductDetails return JSON object. Merges color option
|
||||
func GetProductDetails(productId string) (models.TrendyolProductDetailModel, error) {
|
||||
|
||||
var response models.TrendyolProductDetailResponse
|
||||
productDetailModel := models.TrendyolProductDetailModel{}
|
||||
|
||||
// set linearVariants false to get variants
|
||||
url := "https://public.trendyol.com/discovery-web-productgw-service/api/productDetail/" + productId + "?storefrontId=1&culture=tr-TR&linearVariants=false"
|
||||
|
||||
body, err := helper.SendRequest("GET", url, nil, "")
|
||||
|
||||
if err != nil {
|
||||
return productDetailModel, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &response)
|
||||
|
||||
if err != nil {
|
||||
helper.Error(err)
|
||||
return productDetailModel, err
|
||||
}
|
||||
|
||||
productDetailModel = response.Result
|
||||
|
||||
return productDetailModel, nil
|
||||
}
|
||||
|
||||
// getProductDetailWithOptions returns JSON with variants
|
||||
func (l LinkParser) GetProductDetailWithOptions(productId, productGroupId int) (models.Product, error) {
|
||||
|
||||
primaryProductDetail, err := GetProductDetails(strconv.Itoa(productId))
|
||||
|
||||
if err != nil {
|
||||
return models.Product{}, err
|
||||
}
|
||||
|
||||
productDetailJSON := CreateJSONFromModel(primaryProductDetail)
|
||||
|
||||
if productDetailJSON != nil {
|
||||
colorVariants, err := GetProductColorVariants(productGroupId)
|
||||
|
||||
var colorVariantsJson []map[string]interface{}
|
||||
|
||||
if err != nil {
|
||||
return models.Product{}, err
|
||||
}
|
||||
|
||||
// get the color variant products
|
||||
for _, slicingAttribute := range colorVariants.Result.SlicingAttributes {
|
||||
for _, attribute := range slicingAttribute.Attributes {
|
||||
for _, content := range attribute.Contents {
|
||||
// Don't fetch primary product again.
|
||||
if content.ID != primaryProductDetail.ID {
|
||||
productVariantDetail, errGPD := GetProductDetails(strconv.Itoa(content.ID))
|
||||
if errGPD == nil {
|
||||
productVariantJSON := CreateJSONFromModel(productVariantDetail)
|
||||
if productVariantJSON != nil {
|
||||
colorVariantsJson = append(colorVariantsJson, productVariantJSON)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Color variant count
|
||||
if len(slicingAttribute.Attributes) > 0 {
|
||||
productDetailJSON["color_variant_count"] = len(slicingAttribute.Attributes)
|
||||
productDetailJSON["color_variants"] = colorVariantsJson
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
jsonString, _ := json.Marshal(productDetailJSON)
|
||||
|
||||
// convert json to struct
|
||||
converted := models.Product{}
|
||||
json.Unmarshal(jsonString, &converted)
|
||||
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
// getProductColorVariants returns color options of product
|
||||
func GetProductColorVariants(productGroupId int) (models.TrendyolProductVariantsResponse, error) {
|
||||
|
||||
url := "https://public.trendyol.com/discovery-web-productgw-service/api/productGroup/" + strconv.Itoa(productGroupId) + "?storefrontId=1&culture=tr-TR"
|
||||
|
||||
var response models.TrendyolProductVariantsResponse
|
||||
|
||||
body, err := helper.SendRequest("GET", url, nil, "")
|
||||
|
||||
if err != nil {
|
||||
return response, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &response)
|
||||
|
||||
if err != nil {
|
||||
helper.Error(err)
|
||||
return response, err
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// createJSONFromModel creates json from [trendyol.TrendyolProductDetailModel] and returns
|
||||
func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]interface{} {
|
||||
|
||||
json := map[string]interface{}{}
|
||||
|
||||
// get weight from categories (Stored in helper array (RAM))
|
||||
weight := helper.GetCategoryWeight(model.Category.BeautifiedName)
|
||||
productGroupId := strconv.Itoa(model.ProductGroupID)
|
||||
|
||||
json["_id"] = productGroupId
|
||||
json["product_group_id"] = productGroupId
|
||||
json["vendor"] = "trendyol"
|
||||
json["sku"] = "p-" + strconv.Itoa(model.ID)
|
||||
json["product_number"] = strconv.Itoa(model.ID)
|
||||
json["product_code"] = model.ProductCode
|
||||
json["name"] = model.Name
|
||||
json["sellable"] = model.IsSellable
|
||||
json["favorite_count"] = model.FavoriteCount
|
||||
json["weight"] = weight
|
||||
json["name_with_product_code"] = model.NameWithProductCode
|
||||
json["url_key"] = "https://www.trendyol.com" + model.URL
|
||||
json["images"] = model.Images
|
||||
json["brand"] = model.Brand.Name
|
||||
json["cinsiyet"] = model.Gender.Name
|
||||
json["description"] = model.Description
|
||||
json["descriptions"] = model.ContentDescriptions
|
||||
json["short_description"] = model.Description
|
||||
// nested structure
|
||||
json["price"] = make(map[string]interface{})
|
||||
json["price"].(map[string]interface{})["originalPrice"] = model.Price.OriginalPrice
|
||||
json["price"].(map[string]interface{})["sellingPrice"] = model.Price.SellingPrice
|
||||
json["price"].(map[string]interface{})["discountedPrice"] = model.Price.DiscountedPrice
|
||||
|
||||
attrLen := len(model.Attributes)
|
||||
|
||||
if attrLen != 0 {
|
||||
for i := 0; i < attrLen; i++ {
|
||||
attribute := model.Attributes[i]
|
||||
if attribute.Key.Name == "Renk" {
|
||||
json["color"] = attribute.Value.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// set categories with value 1
|
||||
json["categories"] = []int{1}
|
||||
|
||||
attributes := make([]map[string]string, 0)
|
||||
for _, attr := range model.Attributes {
|
||||
var re = regexp.MustCompile(`/[^A-Z0-9]/ig`)
|
||||
keyStr := re.ReplaceAllString(attr.Key.Name, `_`)
|
||||
key := strings.ToLower(keyStr)
|
||||
attribute := map[string]string{
|
||||
key: attr.Value.Name,
|
||||
}
|
||||
|
||||
attributes = append(attributes, attribute)
|
||||
}
|
||||
|
||||
json["attributes"] = attributes
|
||||
|
||||
var variants []models.Variant
|
||||
|
||||
// if show variants, then it is configurable product.
|
||||
if model.ShowVariants {
|
||||
|
||||
for i := 0; i < len(model.Variants); i++ {
|
||||
variant := model.Variants[i]
|
||||
|
||||
if variant.Sellable {
|
||||
stockType := reflect.TypeOf(variant.Stock)
|
||||
|
||||
if stockType == nil {
|
||||
variants = append(variants, variant)
|
||||
} else {
|
||||
// scrape is via link parse
|
||||
// we need to parse other variants
|
||||
|
||||
// convert stock to int
|
||||
stock, ok := variant.Stock.(float64)
|
||||
|
||||
if ok {
|
||||
if stock > 0 {
|
||||
variants = append(variants, variant)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
for i := 0; i < len(model.AlternativeVariants); i++ {
|
||||
alternativeVariant := model.AlternativeVariants[i]
|
||||
|
||||
stockType := reflect.TypeOf(alternativeVariant.Stock)
|
||||
|
||||
if stockType == nil && len(variants) > 0 {
|
||||
|
||||
// get the first variant for attribute info
|
||||
fv := variants[0]
|
||||
|
||||
variant := models.Variant{
|
||||
AttributeID: fv.AttributeID,
|
||||
AttributeName: fv.AttributeName,
|
||||
AttributeType: fv.AttributeType,
|
||||
AttributeValue: alternativeVariant.AttributeValue,
|
||||
Price: alternativeVariant.Price,
|
||||
}
|
||||
variants = append(variants, variant)
|
||||
}
|
||||
}
|
||||
|
||||
json["size_variants"] = variants
|
||||
}
|
||||
|
||||
if model.ShowVariants && len(variants) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return json
|
||||
}
|
||||
|
|
@ -19,6 +19,9 @@ github.com/jinzhu/now
|
|||
# github.com/joho/godotenv v1.4.0
|
||||
## explicit; go 1.12
|
||||
github.com/joho/godotenv
|
||||
# github.com/leesper/couchdb-golang v1.2.1
|
||||
## explicit
|
||||
github.com/leesper/couchdb-golang
|
||||
# gorm.io/driver/mysql v1.3.5
|
||||
## explicit; go 1.14
|
||||
gorm.io/driver/mysql
|
||||
|
|
|
|||
Loading…
Reference in New Issue