link parse test ready

controllers/ImportController.go

@@ -1,4 +1,4 @@
-package importer
+package controller
 
 import (
 	gm "db_service/gorm_models"
@@ -18,14 +18,16 @@ import (
 	"gorm.io/gorm"
 )
 
-var mainCategories []gm.Category
+var (
-var baza *gorm.DB
+	mainCategories               []gm.Category
-var mainImportWG, famAndSellerWG sync.WaitGroup
+	baza                         *gorm.DB
-var families []gm.AttributeFamily
+	mainImportWG, famAndSellerWG sync.WaitGroup
-var sellers = make(map[string]gm.MarketplaceSeller)
+	families                     []gm.AttributeFamily
-var attributesMap = make(map[string]gm.Attribute)
+	sellers                      = make(map[string]gm.MarketplaceSeller)
+	attributesMap                = make(map[string]gm.Attribute)
+)
 
-func Start(w http.ResponseWriter, route *http.Request) {
+func StartImport(w http.ResponseWriter, route *http.Request) {
 	start := time.Now()
 
 	r := new(big.Int)
@@ -95,7 +97,7 @@ func Start(w http.ResponseWriter, route *http.Request) {
 	for _, element := range mainCategories {
 		slug := element.Translations[0].Slug
 
-		go startImport("ty_db_"+slug, baza)
+		go importCategoryProducts("ty_db_"+slug, baza)
 
 		// fmt.Println(<-result)
 	}
@@ -113,7 +115,7 @@ func Start(w http.ResponseWriter, route *http.Request) {
 
 }
 
-func startImport(dbName string, db *gorm.DB) {
+func importCategoryProducts(dbName string, db *gorm.DB) {
 	defer mainImportWG.Done()
 
 	dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName)
@@ -135,7 +137,7 @@ func startImport(dbName string, db *gorm.DB) {
 
 			skip += limit
 
-			body, err := helper.SendRequest("GET", url, nil, "", true)
+			body, err := helper.SendRequest("GET", url, nil, "")
 
 			if err != nil {
 				fmt.Println(err.Error())
@@ -151,7 +153,7 @@ func startImport(dbName string, db *gorm.DB) {
 
 			//itearate 100 row products
 			for _, element := range response.Rows {
-				importProduct(element.Doc, db)
+				ImportProduct(element.Doc, db)
 			}
 		}
 	} else {
@@ -166,7 +168,7 @@ func getTotalDocumentCount(db string) int {
 
 	url := os.Getenv("couch_db_source") + db
 
-	body, err := helper.SendRequest("GET", url, nil, "", true)
+	body, err := helper.SendRequest("GET", url, nil, "")
 
 	if err != nil {
 		log.Println(err.Error())
@@ -206,7 +208,7 @@ func getCats(db *gorm.DB, catIDs []int) ([]gm.Category, string, error) {
 	return categories, keywords, nil
 }
 
-func importProduct(product models.Product, db *gorm.DB) {
+func ImportProduct(product models.Product, db *gorm.DB) error {
 
 	famAndSellerWG.Wait() //wait until attribute families and sellers are not get from mysql
 
@@ -214,7 +216,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 
 	if errCat != nil {
 		log.Println(errCat)
-		return
+		return errCat
 	}
 
 	var brand gm.Brand
@@ -262,7 +264,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 	errMainProduct := db.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&iproduct).Error
 	if errMainProduct != nil {
 		log.Println(errMainProduct.Error())
-		return
+		return errMainProduct
 	}
 
 	mainProductFlat.ProductID = iproduct.ID
@@ -273,7 +275,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 
 	if errProductMainFlat != nil {
 		log.Println(errProductMainFlat)
-		return
+		return errProductMainFlat
 	}
 
 	if len(product.ColorVariants) > 0 {
@@ -381,7 +383,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 
 					if errProdVariant != nil {
 						log.Println(errProdVariant)
-						return
+						return errProdVariant
 					}
 
 					flatVariant.ProductID = productVariant.ID
@@ -390,7 +392,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 
 					if errVariant != nil {
 						log.Println(errVariant)
-						return
+						return errVariant
 					}
 
 					mainProductFlat.Variants = append(mainProductFlat.Variants, flatVariant)
@@ -431,7 +433,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 
 				if errProdVariant != nil {
 					log.Println(errProdVariant)
-					return
+					return errProdVariant
 				}
 
 				variantFlat.ProductID = productVariant.ID
@@ -444,7 +446,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 
 				if errVariant != nil {
 					log.Println(errVariant)
-					return
+					return errVariant
 				}
 
 				mainProductFlat.Variants = append(mainProductFlat.Variants, variantFlat)
@@ -540,7 +542,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 
 			if errSizeVar != nil {
 				log.Println(errSizeVar)
-				return
+				return errSizeVar
 			}
 
 			flatVariant.ProductID = sizeVariantProduct.ID
@@ -549,7 +551,7 @@ func importProduct(product models.Product, db *gorm.DB) {
 
 			if errVariant != nil {
 				log.Println(errVariant)
-				return
+				return errVariant
 			}
 
 			mainProductFlat.Variants = append(mainProductFlat.Variants, flatVariant)
@@ -571,6 +573,8 @@ func importProduct(product models.Product, db *gorm.DB) {
 	if errSProduct != nil {
 		log.Println(errSProduct)
 	}
+
+	return nil
 }
 
 func createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct {
@@ -719,3 +723,36 @@ func prepearAttributesWithFlat(data *models.Product) ([]gm.ProductAttributeValue
 
 	return productAttributeValues, flat
 }
+
+//func productAttributesAndFlat(data *models.Product) ([]gm.ProductAttributeValue,gm.ProductFlat){
+//
+//	var description string
+//
+//	for _, desc := range data.Descriptions {
+//		description += "<p>" + desc.Description + "</p>"
+//	}
+//
+//	weight, _ := strconv.ParseFloat(data.Weight, 64)
+//
+//	flat := gm.ProductFlat{
+//		Status:              true,
+//		VisibleIndividually: true,
+//		Name:                data.Name,
+//		Sku:                 data.ProductGroupID,
+//		ProductNumber:       data.ProductNumber,
+//		Description:         description,
+//		UrlKey:              data.ProductGroupID,
+//		Weight:              weight,
+//		FavoritesCount: uint(data.FavoriteCount),
+//	}
+//	return  []gm.ProductAttributeValue{
+//		{AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount},
+//		{AttributeID: attributesMap["source"].ID, TextValue: data.URLKey},
+//		{AttributeID: attributesMap["product_number"].ID, TextValue: data.ProductNumber},
+//		{AttributeID: attributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"},
+//		{AttributeID: attributesMap["weight"].ID, TextValue: data.Weight},
+//		{AttributeID: attributesMap["status"].ID, BooleanValue: true},
+//		{AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true},
+//		{AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
+//	},flat
+//}

controllers/ParseController.go

@@ -0,0 +1,64 @@
+package controller
+
+import (
+	"db_service/gorm_models"
+	"db_service/models"
+	helper "db_service/pkg"
+	"db_service/repositories"
+	"encoding/json"
+	"fmt"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+)
+
+func ParseLink(w http.ResponseWriter, route *http.Request) {
+	start := time.Now()
+
+	link := route.URL.Query().Get("url")
+	helper.Info("link: ", link)
+
+	linkParser := repositories.NewLinkParser(link)
+
+	//todo
+	product, err := linkParser.ParseLink()
+
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		json.NewEncoder(w).Encode(map[string]string{
+			"msg": err.Error(),
+		})
+
+		return
+	}
+
+	jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID)
+
+	if err != nil {
+		helper.Error(err)
+		return
+	}
+
+	baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{})
+
+	if err != nil {
+		helper.Error(err)
+		return
+	}
+
+	ImportProduct(jsonProduct, baza)
+
+	w.WriteHeader(http.StatusOK)
+	json.NewEncoder(w).Encode(map[string]string{
+		"msg":            "Link parsed successfully",
+		"productGroupId": strconv.Itoa(product.ProductGroupID),
+	})
+
+	elapsed := time.Since(start)
+	log.Printf("end parse took %s", elapsed)
+	http.Error(w, fmt.Sprintf("end import took %s", elapsed), http.StatusOK)
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gosimple/slug v1.12.0
 	github.com/joho/godotenv v1.4.0
+	github.com/leesper/couchdb-golang v1.2.1
 	gorm.io/driver/mysql v1.3.5
 	gorm.io/gorm v1.23.8
 )

go.sum

@@ -13,6 +13,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/leesper/couchdb-golang v1.2.1 h1:FqSaTxxT2mVRLbxGQVkZakRRoSzWhPmV8UEKYjA/GWc=
+github.com/leesper/couchdb-golang v1.2.1/go.mod h1:OU3FDAM3mazHx15oi8Hm+egTMneBUqepwnh0LuBSH54=
 gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY=
 gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
 gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=

main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	importer "db_service/controllers"
+	"db_service/controllers"
 	"github.com/gorilla/mux"
 	"github.com/joho/godotenv"
 	"log"
@@ -23,17 +23,12 @@ func init() {
 func main() {
 
 	route := mux.NewRouter()
-	route.HandleFunc("/init-importer", importer.Start)
+	route.HandleFunc("/init-importer", controller.StartImport)
+	route.HandleFunc("/parse-link", controller.ParseLink)
 	err := http.ListenAndServe(os.Getenv("port"), route)
 
 	if err != nil {
 		log.Fatal("error: ", err)
 	}
-	//err := importer.Start()
-	//if err != nil {
-	//	fmt.Println("err: ", err.Error())
-	//}
-
-	// dene()
 
 }

models/category.go

@@ -16,3 +16,15 @@ type Category struct {
 type BagistoResponse struct {
 	Data []Category
 }
+type CouchCategory struct {
+	Rev       string `json:"_rev"`
+	CreatedAt string `json:"createdAt"`
+	ID        string `json:"id"`
+	Name      string `json:"name"`
+	Order     string `json:"order"`
+	ParentID  string `json:"parent_id"`
+	SargaID   string `json:"sarga_id"`
+	Slug      string `json:"slug"`
+	UpdatedAt string `json:"updatedAt"`
+	Weight    string `json:"weight"`
+}

models/product.go

@@ -77,3 +77,237 @@ type Row struct {
 type BagistoModelResponse struct {
 	Rows []Row `json:"rows"`
 }
+
+type TrendyolProductDetailResponse struct {
+	IsSuccess  bool                       `json:"isSuccess"`
+	StatusCode int                        `json:"statusCode"`
+	Error      interface{}                `json:"error"`
+	Result     TrendyolProductDetailModel `json:"result"`
+	Headers    struct {
+		Tysidecarcachable string `json:"tysidecarcachable"`
+	} `json:"headers"`
+}
+
+type TrendyolProductDetailModel struct {
+	AlternativeVariants []AlternativeVariant `json:"alternativeVariants"`
+	Attributes          []struct {
+		Key struct {
+			Name string `json:"name"`
+			ID   int    `json:"id"`
+		} `json:"key"`
+		Value struct {
+			Name string `json:"name"`
+			ID   int    `json:"id"`
+		} `json:"value"`
+		Starred bool `json:"starred"`
+	} `json:"attributes"`
+	Variants       []Variant     `json:"variants"`
+	OtherMerchants []interface{} `json:"otherMerchants"`
+	Campaign       struct {
+		ID                 int    `json:"id"`
+		Name               string `json:"name"`
+		StartDate          string `json:"startDate"`
+		EndDate            string `json:"endDate"`
+		IsMultipleSupplied bool   `json:"isMultipleSupplied"`
+		StockTypeID        int    `json:"stockTypeId"`
+		URL                string `json:"url"`
+		ShowTimer          bool   `json:"showTimer"`
+	} `json:"campaign"`
+	Category struct {
+		ID             int    `json:"id"`
+		Name           string `json:"name"`
+		Hierarchy      string `json:"hierarchy"`
+		Refundable     bool   `json:"refundable"`
+		BeautifiedName string `json:"beautifiedName"`
+		IsVASEnabled   bool   `json:"isVASEnabled"`
+	} `json:"category"`
+	Brand struct {
+		IsVirtual      bool   `json:"isVirtual"`
+		BeautifiedName string `json:"beautifiedName"`
+		ID             int    `json:"id"`
+		Name           string `json:"name"`
+		Path           string `json:"path"`
+	} `json:"brand"`
+	Color     string `json:"color"`
+	MetaBrand struct {
+		ID             int    `json:"id"`
+		Name           string `json:"name"`
+		BeautifiedName string `json:"beautifiedName"`
+		IsVirtual      bool   `json:"isVirtual"`
+		Path           string `json:"path"`
+	} `json:"metaBrand"`
+	ShowVariants         bool          `json:"showVariants"`
+	ShowSexualContent    bool          `json:"showSexualContent"`
+	BrandCategoryBanners []interface{} `json:"brandCategoryBanners"`
+	AllVariants          []struct {
+		ItemNumber int     `json:"itemNumber"`
+		Value      string  `json:"value"`
+		InStock    bool    `json:"inStock"`
+		Currency   string  `json:"currency"`
+		Barcode    string  `json:"barcode"`
+		Price      float64 `json:"price"`
+	} `json:"allVariants"`
+	OtherMerchantVariants []interface{} `json:"otherMerchantVariants"`
+	InstallmentBanner     interface{}   `json:"installmentBanner"`
+	IsVasEnabled          bool          `json:"isVasEnabled"`
+	OriginalCategory      struct {
+		ID             int    `json:"id"`
+		Name           string `json:"name"`
+		Hierarchy      string `json:"hierarchy"`
+		Refundable     bool   `json:"refundable"`
+		BeautifiedName string `json:"beautifiedName"`
+		IsVASEnabled   bool   `json:"isVASEnabled"`
+	} `json:"originalCategory"`
+	Landings            []interface{} `json:"landings"`
+	ID                  int           `json:"id"`
+	ProductCode         string        `json:"productCode"`
+	Name                string        `json:"name"`
+	NameWithProductCode string        `json:"nameWithProductCode"`
+	Description         string        `json:"description"`
+	ContentDescriptions []struct {
+		Description string `json:"description"`
+		Bold        bool   `json:"bold"`
+	} `json:"contentDescriptions"`
+	ProductGroupID int    `json:"productGroupId"`
+	Tax            int    `json:"tax"`
+	BusinessUnit   string `json:"businessUnit"`
+	MaxInstallment int    `json:"maxInstallment"`
+	Gender         struct {
+		Name string `json:"name"`
+		ID   int    `json:"id"`
+	} `json:"gender"`
+	URL              string   `json:"url"`
+	Images           []string `json:"images"`
+	IsSellable       bool     `json:"isSellable"`
+	IsBasketDiscount bool     `json:"isBasketDiscount"`
+	HasStock         bool     `json:"hasStock"`
+	Price            Price    `json:"price"`
+	IsFreeCargo      bool     `json:"isFreeCargo"`
+	Promotions       []struct {
+		PromotionRemainingTime string `json:"promotionRemainingTime"`
+		Type                   int    `json:"type"`
+		Text                   string `json:"text"`
+		ID                     int    `json:"id"`
+		Link                   string `json:"link"`
+	} `json:"promotions"`
+	Merchant struct {
+		IsSearchableMerchant bool          `json:"isSearchableMerchant"`
+		Stickers             []interface{} `json:"stickers"`
+		ID                   int           `json:"id"`
+		Name                 string        `json:"name"`
+		OfficialName         string        `json:"officialName"`
+		CityName             string        `json:"cityName"`
+		TaxNumber            string        `json:"taxNumber"`
+		SellerScore          float64       `json:"sellerScore"`
+		SellerScoreColor     string        `json:"sellerScoreColor"`
+		DeliveryProviderName string        `json:"deliveryProviderName"`
+		SellerLink           string        `json:"sellerLink"`
+	} `json:"merchant"`
+	DeliveryInformation struct {
+		IsRushDelivery bool   `json:"isRushDelivery"`
+		DeliveryDate   string `json:"deliveryDate"`
+	} `json:"deliveryInformation"`
+	CargoRemainingDays int  `json:"cargoRemainingDays"`
+	IsMarketplace      bool `json:"isMarketplace"`
+	ProductStamps      []struct {
+		Type          string  `json:"type"`
+		ImageURL      string  `json:"imageUrl"`
+		Position      string  `json:"position"`
+		AspectRatio   float64 `json:"aspectRatio"`
+		Priority      int     `json:"priority"`
+		PriceTagStamp bool    `json:"priceTagStamp,omitempty"`
+	} `json:"productStamps"`
+	HasHTMLContent    bool   `json:"hasHtmlContent"`
+	FavoriteCount     int    `json:"favoriteCount"`
+	UxLayout          string `json:"uxLayout"`
+	IsDigitalGood     bool   `json:"isDigitalGood"`
+	IsRunningOut      bool   `json:"isRunningOut"`
+	ScheduledDelivery bool   `json:"scheduledDelivery"`
+	RatingScore       struct {
+		AverageRating     float64 `json:"averageRating"`
+		TotalRatingCount  int     `json:"totalRatingCount"`
+		TotalCommentCount int     `json:"totalCommentCount"`
+	} `json:"ratingScore"`
+	ShowStarredAttributes    bool   `json:"showStarredAttributes"`
+	ReviewsURL               string `json:"reviewsUrl"`
+	QuestionsURL             string `json:"questionsUrl"`
+	SellerQuestionEnabled    bool   `json:"sellerQuestionEnabled"`
+	SizeExpectationAvailable bool   `json:"sizeExpectationAvailable"`
+	CrossPromotionAward      struct {
+		AwardType  interface{} `json:"awardType"`
+		AwardValue interface{} `json:"awardValue"`
+		ContentID  int         `json:"contentId"`
+		MerchantID int         `json:"merchantId"`
+	} `json:"crossPromotionAward"`
+	RushDeliveryMerchantListingExist bool `json:"rushDeliveryMerchantListingExist"`
+	LowerPriceMerchantListingExist   bool `json:"lowerPriceMerchantListingExist"`
+	ShowValidFlashSales              bool `json:"showValidFlashSales"`
+	ShowExpiredFlashSales            bool `json:"showExpiredFlashSales"`
+	WalletRebate                     struct {
+		MinPrice    int     `json:"minPrice"`
+		MaxPrice    int     `json:"maxPrice"`
+		RebateRatio float64 `json:"rebateRatio"`
+	} `json:"walletRebate"`
+	IsArtWork bool `json:"isArtWork"`
+}
+
+type TrendyolProductVariantsResponse struct {
+	IsSuccess  bool        `json:"isSuccess"`
+	StatusCode int         `json:"statusCode"`
+	Error      interface{} `json:"error"`
+	Result     struct {
+		SlicingAttributes []struct {
+			Brand struct {
+				BeautifiedName string `json:"beautifiedName"`
+				ID             int    `json:"id"`
+				Name           string `json:"name"`
+				IsVirtual      bool   `json:"isVirtual"`
+				Path           string `json:"path"`
+			} `json:"brand"`
+			Attributes []struct {
+				Contents []struct {
+					URL      string `json:"url"`
+					ID       int    `json:"id"`
+					ImageURL string `json:"imageUrl"`
+					Name     string `json:"name"`
+					Price    struct {
+						DiscountedPrice struct {
+							Text  string  `json:"text"`
+							Value float64 `json:"value"`
+						} `json:"discountedPrice"`
+						OriginalPrice struct {
+							Text  string  `json:"text"`
+							Value float64 `json:"value"`
+						} `json:"originalPrice"`
+						SellingPrice struct {
+							Text  string  `json:"text"`
+							Value float64 `json:"value"`
+						} `json:"sellingPrice"`
+					} `json:"price"`
+				} `json:"contents"`
+				Name           string `json:"name"`
+				BeautifiedName string `json:"beautifiedName"`
+			} `json:"attributes"`
+			Type        string `json:"type"`
+			DisplayName string `json:"displayName"`
+			Order       int    `json:"order"`
+			DisplayType int    `json:"displayType"`
+		} `json:"slicingAttributes"`
+	} `json:"result"`
+	Headers struct {
+		Tysidecarcachable string `json:"tysidecarcachable"`
+	} `json:"headers"`
+}
+type AlternativeVariant struct {
+	AttributeValue           string      `json:"attributeValue"`
+	AttributeBeautifiedValue string      `json:"attributeBeautifiedValue"`
+	CampaignID               int         `json:"campaignId"`
+	MerchantID               int         `json:"merchantId"`
+	URLQuery                 string      `json:"urlQuery"`
+	ListingID                string      `json:"listingId"`
+	ItemNumber               int         `json:"itemNumber"`
+	Barcode                  string      `json:"barcode"`
+	Stock                    interface{} `json:"stock"`
+	Quantity                 int         `json:"quantity"`
+	Price                    Price       `json:"price"`
+}

pkg/global.go

@@ -0,0 +1,12 @@
+package helper
+
+import (
+	"db_service/models"
+	"github.com/leesper/couchdb-golang"
+)
+
+// CdbServer is global couchdb server
+var CdbServer *couchdb.Server
+
+// Categories stores all categories and their weight
+var Categories []models.CouchCategory = []models.CouchCategory{}

pkg/helper.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"time"
 )
 
@@ -64,3 +65,22 @@ func CheckDBExists(endpoint string) bool {
 		return false
 	}
 }
+
+func GetCategoryWeight(slug string) string {
+
+	db, err := CdbServer.Get(os.Getenv("db_ty_categories"))
+
+	if err != nil {
+		Error("can not get db. Err: " + err.Error())
+		return "0.5"
+	}
+
+	doc, err := db.Get(slug, nil)
+
+	if err != nil {
+		Error("can not get by slug. Err: " + err.Error())
+		return "0.5"
+	}
+
+	return doc["weight"].(string)
+}

pkg/log.go

@@ -0,0 +1,86 @@
+package helper
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"time"
+)
+
+type Level int
+
+var (
+	DefaultPrefix      = ""
+	DefaultCallerDepth = 2
+
+	logger     *log.Logger
+	logPrefix  = ""
+	levelFlags = []string{"DEBUG", "INFO", "WARN", "ERROR", "FATAL"}
+)
+
+const (
+	DEBUG Level = iota
+	INFO
+	WARNING
+	ERROR
+	FATAL
+)
+
+// Setup initialize the log instance
+func Setup() {
+	logger = log.New(os.Stdout, DefaultPrefix, log.Flags()&^(log.Ldate|log.Ltime))
+}
+
+// Debug output logs at debug level
+func Debug(v ...interface{}) {
+	setPrefix(DEBUG)
+	logger.Println(v...)
+}
+
+// Info output logs at info level
+func Info(v ...interface{}) {
+	setPrefix(INFO)
+	logger.Println(v...)
+}
+
+// Warn output logs at warn level
+func Warn(v ...interface{}) {
+	setPrefix(WARNING)
+	logger.Println(v...)
+}
+
+// Error output logs at error level
+func Error(v ...interface{}) {
+	prefix := setPrefix(ERROR)
+	logger.Println(v...)
+
+	prefix.msg = v
+}
+
+// Fatal output logs at fatal level
+func Fatal(v ...interface{}) {
+	setPrefix(FATAL)
+	logger.Fatalln(v...)
+}
+
+// setPrefix set the prefix of the log output
+func setPrefix(level Level) prefix {
+	_, file, line, ok := runtime.Caller(DefaultCallerDepth)
+	if ok {
+		logPrefix = fmt.Sprintf("[%s:%d] [%s] ", filepath.Base(file), line, levelFlags[level])
+	} else {
+		logPrefix = fmt.Sprintf("[%s] ", levelFlags[level])
+	}
+
+	logPrefix += "| " + time.Now().Format("01.02.2006 15:04:05") + " | "
+
+	logger.SetPrefix(logPrefix)
+	return prefix{prefix: logPrefix}
+}
+
+type prefix struct {
+	prefix string
+	msg    interface{}
+}

pkg/request.go

@@ -5,16 +5,15 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net"
 	"net/http"
 	"time"
 )
 
-func SendRequest(method string, endpoint string, values []byte, authKey string, isCouchDbReq bool) ([]byte, error) {
+func SendRequest(method string, endpoint string, values []byte, authKey string) ([]byte, error) {
 
 	const ConnectMaxWaitTime = 30 * time.Second
-	const RequestMaxWaitTime = 10 * time.Minute
+	const RequestMaxWaitTime = 60 * time.Second
 
 	client := http.Client{
 		Transport: &http.Transport{
@@ -32,7 +31,7 @@ func SendRequest(method string, endpoint string, values []byte, authKey string,
 	req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewBuffer(values))
 	req.Proto = "HTTP/2.0"
 	if err != nil {
-		log.Println(err.Error())
+		Error(err)
 		return emptyBody, err
 	}
 
@@ -49,45 +48,27 @@ func SendRequest(method string, endpoint string, values []byte, authKey string,
 		if response != nil {
 			response.Body.Close()
 		}
-		log.Println(err.Error())
+		Error(err)
 		return emptyBody, err
 	} else if err != nil {
 		if response != nil {
 			response.Body.Close()
 		}
-		log.Println(err.Error())
+		Error(err)
 		return emptyBody, err
 	}
-
+	if response.StatusCode != http.StatusOK {
-	// 200 OK
+		if response != nil {
-	// 201 Created
+			response.Body.Close()
-	// 202 Accepted
-	// 404 Not Found
-	// 409 Conflict
-
-	if isCouchDbReq {
-		// fmt.Printf("responseStatusCode: %d\n", response.StatusCode)
-		if response.StatusCode == http.StatusNotFound {
-			if response != nil {
-				response.Body.Close()
-			}
-			return emptyBody, nil
-		}
-		// TODO: handle conflict error
-	} else {
-		if response.StatusCode != http.StatusOK {
-			if response != nil {
-				response.Body.Close()
-			}
-			err := fmt.Errorf("response: code: %d, body: %v", response.StatusCode, response.Body)
-			log.Println(err.Error())
-			return emptyBody, err
 		}
+		err := fmt.Errorf("response: code: %d, body: %v", response.StatusCode, response.Body)
+		Error(err)
+		return emptyBody, err
 	}
 
 	body, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		log.Println(err.Error())
+		Error(err)
 		if response != nil {
 			response.Body.Close()
 		}
@@ -99,3 +80,20 @@ func SendRequest(method string, endpoint string, values []byte, authKey string,
 
 	return body, nil
 }
+func NewHttpClient() (http.Client, context.Context) {
+	const ConnectMaxWaitTime = 80 * time.Second
+	const RequestMaxWaitTime = 120 * time.Second
+
+	client := http.Client{
+		Transport: &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout: ConnectMaxWaitTime,
+			}).DialContext,
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), RequestMaxWaitTime)
+	defer cancel()
+
+	return client, ctx
+}

repositories/ParseRepository.go

@@ -0,0 +1,325 @@
+package repositories
+
+import (
+	"db_service/models"
+	helper "db_service/pkg"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type LinkParser struct {
+	link string
+}
+
+func NewLinkParser(link string) LinkParser {
+	return LinkParser{link: link}
+}
+
+func (l LinkParser) ParseLink() (models.TrendyolProductDetailModel, error) {
+
+	helper.Info("link: ", l.link)
+
+	productId := ""
+
+	if isShortLink(l.link) {
+		productId = getProductIdFromShortLink(l.link)
+	} else {
+		productId = getProductIdFromLink(l.link)
+	}
+
+	if len(productId) == 0 {
+		parseErr := errors.New("can not parse product id")
+		helper.Error(parseErr)
+		return models.TrendyolProductDetailModel{}, parseErr
+	}
+
+	helper.Info("productId: ", productId)
+
+	return GetProductDetails(productId)
+}
+
+func getProductIdFromShortLink(shortLink string) string {
+
+	var productId string
+
+	client, _ := helper.NewHttpClient()
+	req, err := http.NewRequest("GET", shortLink, nil)
+	req.Proto = "HTTP/2.0"
+	if err != nil {
+		helper.Error(err)
+		return ""
+	}
+
+	q := req.URL.Query()
+	req.URL.RawQuery = q.Encode()
+	req.Header.Set("Connection", "close")
+	req = req.WithContext(req.Context())
+	req.Close = true
+	client.CloseIdleConnections()
+
+	response, err := client.Do(req)
+	if err != nil {
+		helper.Error(err)
+		return productId
+	}
+
+	defer response.Body.Close()
+
+	url := response.Request.URL.Path
+
+	helper.Info("link url: ", url)
+
+	productId = getProductIdFromLink(url)
+
+	helper.Info("productId: ", productId)
+
+	return productId
+}
+
+func getProductIdFromLink(link string) string {
+	var productId string
+
+	if strings.Contains(link, "?") {
+		link = strings.Split(link, "?")[0]
+	}
+
+	strArr := strings.Split(link, "-")
+	productId = strArr[len(strArr)-1]
+
+	return productId
+}
+
+func isShortLink(link string) bool {
+	return !strings.Contains(link, "trendyol.com")
+}
+
+// GetProductDetails return JSON object. Merges color option
+func GetProductDetails(productId string) (models.TrendyolProductDetailModel, error) {
+
+	var response models.TrendyolProductDetailResponse
+	productDetailModel := models.TrendyolProductDetailModel{}
+
+	// set linearVariants false to get variants
+	url := "https://public.trendyol.com/discovery-web-productgw-service/api/productDetail/" + productId + "?storefrontId=1&culture=tr-TR&linearVariants=false"
+
+	body, err := helper.SendRequest("GET", url, nil, "")
+
+	if err != nil {
+		return productDetailModel, err
+	}
+
+	err = json.Unmarshal(body, &response)
+
+	if err != nil {
+		helper.Error(err)
+		return productDetailModel, err
+	}
+
+	productDetailModel = response.Result
+
+	return productDetailModel, nil
+}
+
+// getProductDetailWithOptions returns JSON with variants
+func (l LinkParser) GetProductDetailWithOptions(productId, productGroupId int) (models.Product, error) {
+
+	primaryProductDetail, err := GetProductDetails(strconv.Itoa(productId))
+
+	if err != nil {
+		return models.Product{}, err
+	}
+
+	productDetailJSON := CreateJSONFromModel(primaryProductDetail)
+
+	if productDetailJSON != nil {
+		colorVariants, err := GetProductColorVariants(productGroupId)
+
+		var colorVariantsJson []map[string]interface{}
+
+		if err != nil {
+			return models.Product{}, err
+		}
+
+		// get the color variant products
+		for _, slicingAttribute := range colorVariants.Result.SlicingAttributes {
+			for _, attribute := range slicingAttribute.Attributes {
+				for _, content := range attribute.Contents {
+					// Don't fetch primary product again.
+					if content.ID != primaryProductDetail.ID {
+						productVariantDetail, errGPD := GetProductDetails(strconv.Itoa(content.ID))
+						if errGPD == nil {
+							productVariantJSON := CreateJSONFromModel(productVariantDetail)
+							if productVariantJSON != nil {
+								colorVariantsJson = append(colorVariantsJson, productVariantJSON)
+							}
+						}
+					}
+				}
+			}
+			// Color variant count
+			if len(slicingAttribute.Attributes) > 0 {
+				productDetailJSON["color_variant_count"] = len(slicingAttribute.Attributes)
+				productDetailJSON["color_variants"] = colorVariantsJson
+			}
+		}
+	}
+
+	jsonString, _ := json.Marshal(productDetailJSON)
+
+	// convert json to struct
+	converted := models.Product{}
+	json.Unmarshal(jsonString, &converted)
+
+	return converted, nil
+}
+
+// getProductColorVariants returns color options of product
+func GetProductColorVariants(productGroupId int) (models.TrendyolProductVariantsResponse, error) {
+
+	url := "https://public.trendyol.com/discovery-web-productgw-service/api/productGroup/" + strconv.Itoa(productGroupId) + "?storefrontId=1&culture=tr-TR"
+
+	var response models.TrendyolProductVariantsResponse
+
+	body, err := helper.SendRequest("GET", url, nil, "")
+
+	if err != nil {
+		return response, err
+	}
+
+	err = json.Unmarshal(body, &response)
+
+	if err != nil {
+		helper.Error(err)
+		return response, err
+	}
+
+	return response, nil
+}
+
+// createJSONFromModel creates json from [trendyol.TrendyolProductDetailModel] and returns
+func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]interface{} {
+
+	json := map[string]interface{}{}
+
+	// get weight from categories (Stored in helper array (RAM))
+	weight := helper.GetCategoryWeight(model.Category.BeautifiedName)
+	productGroupId := strconv.Itoa(model.ProductGroupID)
+
+	json["_id"] = productGroupId
+	json["product_group_id"] = productGroupId
+	json["vendor"] = "trendyol"
+	json["sku"] = "p-" + strconv.Itoa(model.ID)
+	json["product_number"] = strconv.Itoa(model.ID)
+	json["product_code"] = model.ProductCode
+	json["name"] = model.Name
+	json["sellable"] = model.IsSellable
+	json["favorite_count"] = model.FavoriteCount
+	json["weight"] = weight
+	json["name_with_product_code"] = model.NameWithProductCode
+	json["url_key"] = "https://www.trendyol.com" + model.URL
+	json["images"] = model.Images
+	json["brand"] = model.Brand.Name
+	json["cinsiyet"] = model.Gender.Name
+	json["description"] = model.Description
+	json["descriptions"] = model.ContentDescriptions
+	json["short_description"] = model.Description
+	// nested structure
+	json["price"] = make(map[string]interface{})
+	json["price"].(map[string]interface{})["originalPrice"] = model.Price.OriginalPrice
+	json["price"].(map[string]interface{})["sellingPrice"] = model.Price.SellingPrice
+	json["price"].(map[string]interface{})["discountedPrice"] = model.Price.DiscountedPrice
+
+	attrLen := len(model.Attributes)
+
+	if attrLen != 0 {
+		for i := 0; i < attrLen; i++ {
+			attribute := model.Attributes[i]
+			if attribute.Key.Name == "Renk" {
+				json["color"] = attribute.Value.Name
+			}
+		}
+	}
+
+	// set categories with value 1
+	json["categories"] = []int{1}
+
+	attributes := make([]map[string]string, 0)
+	for _, attr := range model.Attributes {
+		var re = regexp.MustCompile(`/[^A-Z0-9]/ig`)
+		keyStr := re.ReplaceAllString(attr.Key.Name, `_`)
+		key := strings.ToLower(keyStr)
+		attribute := map[string]string{
+			key: attr.Value.Name,
+		}
+
+		attributes = append(attributes, attribute)
+	}
+
+	json["attributes"] = attributes
+
+	var variants []models.Variant
+
+	// if show variants, then it is configurable product.
+	if model.ShowVariants {
+
+		for i := 0; i < len(model.Variants); i++ {
+			variant := model.Variants[i]
+
+			if variant.Sellable {
+				stockType := reflect.TypeOf(variant.Stock)
+
+				if stockType == nil {
+					variants = append(variants, variant)
+				} else {
+					// scrape is via link parse
+					// we need to parse other variants
+
+					// convert stock to int
+					stock, ok := variant.Stock.(float64)
+
+					if ok {
+						if stock > 0 {
+							variants = append(variants, variant)
+						}
+					}
+
+				}
+			}
+
+		}
+
+		for i := 0; i < len(model.AlternativeVariants); i++ {
+			alternativeVariant := model.AlternativeVariants[i]
+
+			stockType := reflect.TypeOf(alternativeVariant.Stock)
+
+			if stockType == nil && len(variants) > 0 {
+
+				// get the first variant for attribute info
+				fv := variants[0]
+
+				variant := models.Variant{
+					AttributeID:    fv.AttributeID,
+					AttributeName:  fv.AttributeName,
+					AttributeType:  fv.AttributeType,
+					AttributeValue: alternativeVariant.AttributeValue,
+					Price:          alternativeVariant.Price,
+				}
+				variants = append(variants, variant)
+			}
+		}
+
+		json["size_variants"] = variants
+	}
+
+	if model.ShowVariants && len(variants) == 0 {
+		return nil
+	}
+
+	return json
+}

vendor/modules.txt

@@ -19,6 +19,9 @@ github.com/jinzhu/now
 # github.com/joho/godotenv v1.4.0
 ## explicit; go 1.12
 github.com/joho/godotenv
+# github.com/leesper/couchdb-golang v1.2.1
+## explicit
+github.com/leesper/couchdb-golang
 # gorm.io/driver/mysql v1.3.5
 ## explicit; go 1.14
 gorm.io/driver/mysql