Are you in need of adding a lot +40K of products, attributes, images, slugs, tags, manufacturers in batch?
Then this post should be same useful for you as for me.
In my case I did require to implement a logic for adding more than 40,000 products with attributes, images, etc. feeding data from a 3rd party API. We will add products to all enabled categories and subcategories that are shown on home page/menu.
So less water more sense :) - here is the step by step explanation:
[AuthorizeAdmin]
[Area(AreaNames.Admin)]
public IActionResult GetProgress()
{
//to keep progress
string progress = string.Empty;
var storeScope = _storeContext.ActiveStoreScopeConfiguration;
var directory = "~/wwwroot/images/uploaded/";
string curl = string.Empty;
//Use direct repo access because of cache problem with re-reading data from settings
Core.Data.IRepository<Core.Domain.Configuration.Setting> _settingRepository = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Configuration.Setting>>();
try
{
//get list of rest of categories we should add product for
string ddcat = _settingRepository.Table.SingleOrDefault(s=>s.Name=="DDCategories" && s.StoreId== storeScope)?.Value;
if (string.IsNullOrEmpty(ddCat))
{
//seems this is the first run
var nopallcats = _categoryService.GetAllCategoriesDisplayedOnHomePage(false).Where(c=>c.Published).Select(s => new Category() { ID = s.Id }).ToList();
List<int> finalcategories = new List<int>();
foreach (var c1 in nopallcats)
{
var aa1 = _categoryService.GetAllCategoriesByParentCategoryId((int)c1.ID, true);
if (aa1.Count > 0)
{
foreach (var c2 in aa1)
{
var aa2 = _categoryService.GetAllCategoriesByParentCategoryId(c2.Id, true);
if (aa2.Count > 0 && c2.Published)
{
foreach (var c3 in aa2)
{
var aa3 = _categoryService.GetAllCategoriesByParentCategoryId(c3.Id, true);
if (aa3.Count <= 0 && c3.Published)
{
finalcategories.Add(c3.Id);
}
else
{
//we have only 3 level of sub categories
throw new ApplicationException("Too many subcategories");
}
}
}
else
{
finalcategories.Add(c2.Id);
}
}
}
else
{
finalcategories.Add((int)c1.ID);
}
}
nopallcats = null;
//store remind categories into the settings
_settingService.SetSetting<string>("DDCategories", Newtonsoft.Json.JsonConvert.SerializeObject(finalcategories, Newtonsoft.Json.Formatting.None,
new Newtonsoft.Json.JsonSerializerSettings()
{
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore
}), storeScope, true);
progress = string.Format("Categories: {0} --", finalcategories.Count);
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
}
else
{
progress = "Start working on category...";
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
List<int> categories = Newtonsoft.Json.JsonConvert.DeserializeObject<List<int>>(ddCat);
int products = 0;
//take first category to work on
var rcategory = categories.FirstOrDefault();
//if category id > 0 then lets work on it or exit otherwise
if (rcategory != 0)
{
try
{
List<Price> prices = null;
string responseText = string.Empty;
string json = string.Empty;
System.Net.HttpWebRequest r = null;
System.Net.WebClient webClient = new System.Net.WebClient();
lock (Feeder.fileLock)
{
string pricefile = _fileProvider.GetFileName("dd_prices.json");
pricefile = _fileProvider.Combine(_fileProvider.MapPath(dir2), pricefile);
//if price file does not exists or older than 60 mins then get new
if (!_fileProvider.FileExists(pricefile) || DateTime.UtcNow.Subtract(_fileProvider.GetLastWriteTimeUtc(pricefile)).TotalMinutes > 60)
{
responseText = null;
}
else
{
responseText = _fileProvider.ReadAllText(pricefile, Encoding.UTF8);
prices = Newtonsoft.Json.JsonConvert.DeserializeObject<List<Price>>(responseText);
responseText = progress = "re-read prices from file;";
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
}
if (string.IsNullOrEmpty(responseText) && prices == null)
{
curl = Feeder.apiUrl + Feeder.getPrices;
r = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(curl);
r.ContentType = "application/x-www-form-urlencoded";
r.Method = "POST";
json = "key=" + Feeder.apiKey;
using (System.IO.StreamWriter sw = new System.IO.StreamWriter(r.GetRequestStream()))
{
sw.Write(json);
sw.Flush();
sw.Close();
}
using (var reader = new System.IO.StreamReader(r.GetResponse().GetResponseStream()))
{
responseText = reader.ReadToEnd();
dynamic dynJson3 = Newtonsoft.Json.JsonConvert.DeserializeObject(responseText);
responseText = null;
prices = new List<Price>();
foreach (var pr in dynJson3.prices)
{
//get prices from JSON and save to own JSON file
dynamic price = ((Newtonsoft.Json.Linq.JToken)pr).First;
prices.Add(new Price()
{
code = price.sku,
id = price.id,
price = price.price,
RPrice = price.RPrice,
delivery = price.delivery,
deliveryText = price.deliveryText
});
}
_fileProvider.WriteAllText(pricefile, Newtonsoft.Json.JsonConvert.SerializeObject(prices, Newtonsoft.Json.Formatting.None,
new Newtonsoft.Json.JsonSerializerSettings()
{
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore
}), Encoding.UTF8);
progress = "prices re-downloaded again;";
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
}
}
}
responseText = null;
var dateranges = _dateRangeService.GetAllDeliveryDates();
ProdTemp pt = null;
var category = rcategory;
int count = 0;
int offset = 0, limit = 100, prodProcessed = 0;
var ccc = _categoryService.GetCategoryById(category);
string temprod = _settingRepository.Table.SingleOrDefault(s => s.Name == "DDProdcts" && s.StoreId == storeScope)?.Value;
//create new or re-read last offset object
if (string.IsNullOrEmpty(temprod))
{
pt = new ProdTemp();
pt.StoreID = storeScope;
}
else
{
pt = Newtonsoft.Json.JsonConvert.DeserializeObject<ProdTemp>(temprod);
limit = pt.Limit;
offset = pt.Offset;
count = pt.TotalProducts;
prodProcessed = pt.ProcessedProducts;
storeScope = pt.StoreID.GetValueOrDefault();
}
curl = Feeder.apiUrl + Feeder.getItems;
r = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(curl);
r.ContentType = "application/x-www-form-urlencoded";
r.Method = "POST";
//get 3rd party category id to get products for it
var ccatid = _genericAttributeService.GetAttribute<int>(ccc, Feeder.gAttrID, storeScope);
json = "key=" + Feeder.apiKey + "&limit=" + limit + "&offset=" + offset + "&modifiedDT=" +
DateTime.UtcNow.Date.AddMonths(-6).ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture) +
"&brand=" + "&category=" + ccatid;
using (System.IO.StreamWriter sw = new System.IO.StreamWriter(r.GetRequestStream()))
{
sw.Write(json);
sw.Flush();
sw.Close();
}
responseText = string.Empty;
using (var reader = new System.IO.StreamReader(r.GetResponse().GetResponseStream()))
{
responseText = reader.ReadToEnd();
}
dynamic dynJson2 = Newtonsoft.Json.JsonConvert.DeserializeObject(responseText);
responseText = null;
string swm = string.Empty;
count = int.Parse((string)dynJson2.count);
progress = string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, count, prodProcessed);
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
Core.Domain.Catalog.Product pro = null;
dynamic product;
Price pprice;
var manufRepository = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.Manufacturer>>();
var productRepository = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.Product>>();
var productTagRepository = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.ProductTag>>();
var prodAttrRepo= Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.ProductAttribute>>();
var spAttrRepository = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.SpecificationAttribute>>();
var spOAttrRepository = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.SpecificationAttributeOption>>();
var prodAttrMapRepo = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.ProductAttributeMapping>>();
var prodAttrValRepo = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.ProductAttributeValue>>();
var prodTagMapRepo = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.ProductProductTagMapping>>();
var prodCatMapRepo = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.ProductCategory>>();
var prodSAttrMapRepo = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Catalog.ProductSpecificationAttribute>>();
string entityName = typeof(Core.Domain.Catalog.Product).Name;
var urlRepository = Core.Infrastructure.EngineContext.Current.Resolve<Core.Data.IRepository<Core.Domain.Seo.UrlRecord>>();
//stopwatch to get timing
System.Diagnostics.Stopwatch swc = new System.Diagnostics.Stopwatch();
double totalTimeSec = 0;
foreach (var prodd in dynJson2.items)
{
totalTimeSec = 0;
swm = string.Empty;
_cache.Set<string>("Progress", string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, count, prodProcessed), TimeSpan.FromMinutes(30));
product = ((Newtonsoft.Json.Linq.JToken)prodd).First;
int pid = int.Parse((string)Newtonsoft.Json.Linq.JProperty.FromObject(prodd).Name);
progress = "pprice | " + string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
pprice = prices.SingleOrDefault(p => p.id == pid);
if (pprice == null)
{
pprice = prices.SingleOrDefault(p => p.code == pid.ToString("000000") + "-02");
}
string pname = product.name;
swc.Start();
pro = null;
//different products may have same SKUs so take care of it
pro = _productService.GetProductsBySku(new string[] { (string)product.article }).SingleOrDefault(sk=> sk.Name== pname && sk.ProductCategories.SingleOrDefault(pc=>pc.CategoryId==category)!=null);
swc.Stop();
totalTimeSec += swc.Elapsed.TotalSeconds;
swm = "GetProductBySku: " + swc.Elapsed.TotalSeconds.ToString("0.##");
_cache.Set<string>("Progress", progress + swm, TimeSpan.FromMinutes(30));
swc.Reset();
//we add products if price exists only
if (pprice != null && pro == null)
{
decimal cost = pprice.price;
decimal price = Feeder.CalculatePrice(cost, pprice.RPrice);
if(price<cost)
{
price = pprice.RPrice;
}
//define max price for category to filter by later
if (pt.MaxPrice < price)
pt.MaxPrice = price;
//manufacturer
swc.Start();
int manuid = product.brand;
progress = "manufs | " + string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
_cache.Set<string>("Progress", progress + " " + swm, TimeSpan.FromMinutes(30));
string bname = (string)product.brandName;
var manuf = manufRepository.Table.FirstOrDefault(m => m.Name == bname);
if (manuf == null)
{
manuf = new Core.Domain.Catalog.Manufacturer()
{
Name = product.brandName,
CreatedOnUtc = DateTime.UtcNow,
ManufacturerTemplateId = 1,
MetaDescription = product.brandName,
MetaKeywords = product.brandName,
MetaTitle = product.brandName,
PageSize = 20,
Published = true,
UpdatedOnUtc = DateTime.UtcNow
};
_manufacturerService.InsertManufacturer(manuf);
_storeMappingService.InsertStoreMapping<Core.Domain.Catalog.Manufacturer>(manuf, storeScope);
//store 3rd party manufacturer id to use later
_genericAttributeService.SaveAttribute<int>(manuf, Feeder.gAttrID, manuid);
entityName = typeof(Core.Domain.Catalog.Manufacturer).Name;
//add slug if not exists
if (urlRepository.TableNoTracking.SingleOrDefault(u => u.EntityId == manuf.Id && u.EntityName == entityName && u.LanguageId == 0) == null)
{
var slug1 = _urlRecordService.ValidateSeName(manuf, manuf.Name, manuf.Name, true);
_urlRecordService.InsertUrlRecord(new Core.Domain.Seo.UrlRecord
{
EntityId = manuf.Id,
EntityName = entityName,
LanguageId = 0,
IsActive = true,
Slug = slug1
});
}
}
swc.Stop();
totalTimeSec += swc.Elapsed.TotalSeconds;
swm += ", manufs: " + swc.Elapsed.TotalSeconds.ToString("0.##");
swc.Reset();
//date ranges
swc.Start();
Core.Domain.Shipping.DeliveryDate daterange;
progress = "daterange | " + string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
_cache.Set<string>("Progress", progress + " " + swm, TimeSpan.FromMinutes(30));
{
string dt = (string)pprice.deliveryText;
daterange = dateranges.FirstOrDefault(d => d.Name == dt);
if (daterange == null)
{
daterange = new Core.Domain.Shipping.DeliveryDate()
{
Name = pprice.deliveryText
};
_dateRangeService.InsertDeliveryDate(daterange);
dateranges.Add(daterange);
}
}
swc.Stop();
totalTimeSec += swc.Elapsed.TotalSeconds;
swm += ", daterange: " + swc.Elapsed.TotalSeconds.ToString("0.##");
swc.Reset();
//product
swc.Start();
{
//some formatting and length limitation
string shortd = product.descriptionSnort;
if (shortd.Length > 400)
{
shortd = shortd.Substring(0, 399);
}
if (pname.Length > 400)
{
pname = pname.Substring(0, 399);
}
pro = new Core.Domain.Catalog.Product()
{
ProductType = Core.Domain.Catalog.ProductType.SimpleProduct,
VisibleIndividually = true,
Sku = product.article,
IsShipEnabled = true,
MetaDescription = shortd,
MetaKeywords = shortd,
MetaTitle = pname,
Price = price,
ProductCost = cost,
AdditionalShippingCharge = Feeder.CalculateShipping(price),
CreatedOnUtc = DateTime.UtcNow,
UpdatedOnUtc = ((DateTime)product.modifiedDT).ToUniversalTime(),
Name = pname,
ShortDescription = product.descriptionSnort,
FullDescription = product.description,
BackorderMode = Core.Domain.Catalog.BackorderMode.NoBackorders,
MarkAsNew = true,
AllowBackInStockSubscriptions = true,
AllowCustomerReviews = true,
Published = true,
DeliveryDateId = daterange == null ? 0 : daterange.Id,
LimitedToStores = true, OrderMaximumQuantity=int.MaxValue
,OrderMinimumQuantity=1, DisableBuyButton = true, ProductTemplateId = 1, ProductManufacturers =
{
new Core.Domain.Catalog.ProductManufacturer() { Manufacturer = manuf }
}
};
if (!string.IsNullOrEmpty(pro.FullDescription))
pro.FullDescription = pro.FullDescription.Replace("font-size: 1rem;", string.Empty);
}
{
pro.ProductCategories.Add(new Core.Domain.Catalog.ProductCategory()
{
CategoryId = category
});
}
_productService.InsertProduct(pro);
_genericAttributeService.SaveAttribute<int>(pro, Feeder.gAttrID, pid, storeScope);
_genericAttributeService.SaveAttribute<string>(pro, Feeder.gAttrCode, pprice.code, storeScope);
products++;
swc.Stop();
totalTimeSec += swc.Elapsed.TotalSeconds;
swm += ", product: " + swc.Elapsed.TotalSeconds.ToString("0.##");
swc.Reset();
//tags
swc.Start();
progress = "ttag | " + string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
_cache.Set<string>("Progress", progress + " " + swm, TimeSpan.FromMinutes(30));
foreach (var tt in pro.Name.Split(' ', StringSplitOptions.RemoveEmptyEntries))
{
string tag = tt.ToUpperInvariant().Trim().Trim(Feeder.trimChars);
if (!string.IsNullOrEmpty(tag))
{
if (manufRepository.Table.Any(m => m.Name == tag))
continue;
var ttag = productTagRepository.Table.FirstOrDefault(t => t.Name == tag);
if (ttag == null)
{
ttag = new Core.Domain.Catalog.ProductTag() { Name = tag };
_productTagService.InsertProductTag(ttag);
pro.ProductProductTagMappings.Add(new Core.Domain.Catalog.ProductProductTagMapping()
{
ProductTag = ttag,
});
entityName = typeof(Core.Domain.Catalog.ProductTag).Name;
if (urlRepository.TableNoTracking.SingleOrDefault(u => u.EntityId == ttag.Id && u.EntityName == entityName && u.LanguageId == 0) == null)
{
var slug2 = _urlRecordService.ValidateSeName(ttag, ttag.Name, ttag.Name, true);
_urlRecordService.InsertUrlRecord(new Core.Domain.Seo.UrlRecord
{
EntityId = ttag.Id,
EntityName = entityName,
LanguageId = 0,
IsActive = true,
Slug = slug2
});
}
}
else if (!prodTagMapRepo.Table.Any(a => a.ProductTagId == ttag.Id && a.ProductId == pro.Id))//!pro.ProductProductTagMappings.Any(a => a.ProductTag.Id == ttag.Id))
{
if (!pro.ProductProductTagMappings.Any(a => a.ProductTag.Id == ttag.Id))
{
pro.ProductProductTagMappings.Add(new Core.Domain.Catalog.ProductProductTagMapping()
{
ProductTag = ttag,
});
}
}
}
}
_productService.UpdateProduct(pro);
swc.Stop();
totalTimeSec += swc.Elapsed.TotalSeconds;
swm += ", ttag: " + swc.Elapsed.TotalSeconds.ToString("0.##");
swc.Reset();
_cache.Set<string>("Progress", progress + " " + swm, TimeSpan.FromMinutes(30));
//pictures
swc.Start();
AddPictures(product, pro);
_productService.UpdateProduct(pro);
swc.Stop();
totalTimeSec += swc.Elapsed.TotalSeconds;
swm += ", pictures: " + swc.Elapsed.TotalSeconds.ToString("0.##");
swc.Reset();
_cache.Set<string>("Progress", progress + " " + swm, TimeSpan.FromMinutes(30));
bool colDef = true;
//attributes
swc.Start();
Core.Domain.Catalog.ProductAttribute pattr = null;
Core.Domain.Catalog.SpecificationAttribute spattr = null;
Core.Domain.Catalog.SpecificationAttributeOption spao = null;
List<Core.Domain.Catalog.ProductSpecificationAttribute> prodSAttrMapRepo_later = new List<Core.Domain.Catalog.ProductSpecificationAttribute>();
dynamic chr;
Newtonsoft.Json.Linq.JProperty cjp;
string val, name, rgb;
System.Drawing.Color c;
progress = "pattr | " + string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
_cache.Set<string>("Progress", progress + " " + swm, TimeSpan.FromMinutes(30));
string nlo;
string spaovu;
string spaonu;
foreach (var ch in product.characteristics)
{
chr = ((Newtonsoft.Json.Linq.JToken)ch).First;
cjp = Newtonsoft.Json.Linq.JProperty.FromObject(ch) as Newtonsoft.Json.Linq.JProperty;
val = ((Newtonsoft.Json.Linq.JValue)chr).Value as string;
name = cjp.Name.Trim().Trim(Feeder.trimChars);
if (name.IndexOf('(') >= 0 && name.IndexOf(')') < 0)
{
name += ")";
}
if (name.IndexOf('[') >= 0 && name.IndexOf(']') < 0)
{
name += "]";
}
if (name.IndexOf('{') >= 0 && name.IndexOf('}') < 0)
{
name += "}";
}
nlo = name.ToLowerInvariant();
spaovu = val.ToUpperInvariant();
spaonu = name.ToUpperInvariant();
//work on colors
if (nlo == "цвет" || nlo == "color" || nlo == "колір" || (nlo.Contains("цвет ") || nlo.Contains("color ") || nlo.Contains("колір ")))
{
progress = "pattr | " + string.Format("Categories: {0}, Products: {1}/{2}", categories.Count, (string)dynJson2.count, prodProcessed);
pattr = prodAttrRepo.Table.FirstOrDefault(a => a.Name == name);
if (pattr == null)
{
pattr = new Core.Domain.Catalog.ProductAttribute()
{
Name = name
};
_productAttributeService.InsertProductAttribute(pattr);
}
string[] colors = spaovu.Split(',', StringSplitOptions.RemoveEmptyEntries);
foreach (var sc in colors)
{
rgb = "#ffffff";
var ssc = sc.Replace(" ", string.Empty).Replace("-", string.Empty).Replace("/", string.Empty).Replace("+", string.Empty).ToLowerInvariant();
var sccc = Feeder.colors.SingleOrDefault(s => s.Item2.Contains(ssc));
if (sccc != null)
rgb = sccc.Item1;
else
{
c = System.Drawing.Color.FromName(val);
rgb = "#" + c.R.ToString("X2") + c.G.ToString("X2") + c.B.ToString("X2");
}
if (!prodAttrMapRepo.Table.Any(a => a.ProductAttributeId == pattr.Id && a.ProductId == pro.Id))
if (!pro.ProductAttributeMappings.Any(a => a.ProductAttribute.Id == pattr.Id
&& a.ProductAttributeValues.Any(aa => aa.Name.ToUpper() == spaovu)))
{
pro.ProductAttributeMappings.Add(new Core.Domain.Catalog.ProductAttributeMapping()
{
AttributeControlType = Core.Domain.Catalog.AttributeControlType.ColorSquares,
IsRequired = colors.Length>1,
ProductAttribute = pattr,
ProductAttributeValues =
{
new Core.Domain.Catalog.ProductAttributeValue
{
AttributeValueType = Core.Domain.Catalog.AttributeValueType.Simple,
Name = sc, IsPreSelected=colDef, ColorSquaresRgb=rgb
}
}
});
colDef = false;
}
}
}
else //work on specification attributes
{
progress = "spattr | " + string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
spattr = spAttrRepository.Table.FirstOrDefault(a => a.Name == name);
spao = null;
if (spattr == null)
{
spattr = new Core.Domain.Catalog.SpecificationAttribute
{
Name = name
};
_specificationAttributeService.InsertSpecificationAttribute(spattr);
spao = new Core.Domain.Catalog.SpecificationAttributeOption()
{
Name = val,
SpecificationAttribute = spattr
};
_specificationAttributeService.InsertSpecificationAttributeOption(spao);
}
else
{
progress = "spao | " + string.Format("Categories: {0}, Products: {1}/{2}", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
spao = spOAttrRepository.Table.FirstOrDefault(s => s.SpecificationAttributeId == spattr.Id && s.Name == val);
if (spao == null)
{
progress = "!!!" + progress;
_cache.Set<string>("Progress", progress + " " + swm, TimeSpan.FromMinutes(30));
{
spao = new Core.Domain.Catalog.SpecificationAttributeOption()
{
SpecificationAttribute = spattr,
Name = val
};
_specificationAttributeService.InsertSpecificationAttributeOption(spao);
}
}
}
if (!prodSAttrMapRepo.Table.Any(a => a.SpecificationAttributeOptionId == spao.Id &&
a.ProductId == pro.Id))
if (!pro.ProductSpecificationAttributes.Any(a => a.SpecificationAttributeOptionId == spao.Id
&& a.SpecificationAttributeOption.SpecificationAttributeId == spattr.Id))
{
pro.ProductSpecificationAttributes.Add(new Core.Domain.Catalog.ProductSpecificationAttribute()
{
//disable filtering by SKU or whatever you need
AllowFiltering = !excludeParams.Any(a => nlo.ToUpper().Contains(a.ToUpper())),
ShowOnProductPage = nlo.ToUpper() != excludeParams[0].ToUpper() && nlo.ToUpper() != excludeParams[1].ToUpper(),
SpecificationAttributeOption = spao,
AttributeType = Core.Domain.Catalog.SpecificationAttributeType.Option
});
}
}
_productService.UpdateProduct(pro);
}
entityName = typeof(Core.Domain.Catalog.Product).Name;
var slugp = _urlRecordService.ValidateSeName(pro, pro.Name, pro.Name, true);
urlRepository.Insert(new Core.Domain.Seo.UrlRecord
{
EntityId = pro.Id,
EntityName = entityName,
LanguageId = 0,
IsActive = true,
Slug = slugp
});
swc.Stop();
totalTimeSec += swc.Elapsed.TotalSeconds;
swm += ", pattr: " + swc.Elapsed.TotalSeconds.ToString("0.##");
swc.Reset();
_cache.Set<string>("Progress", totalTimeSec.ToString("0.###") + ":" + progress + " " + swm, TimeSpan.FromMinutes(30));
}
else if (pro != null)
{
//do whatever you need if product already exists
}
prodProcessed++;
offset++;
pt.Limit = limit;
pt.Offset = offset;
pt.ProcessedProducts = prodProcessed;
pt.TotalProducts = count;
_settingService.SetSetting<string>("DDProdcts", Newtonsoft.Json.JsonConvert.SerializeObject(pt, Newtonsoft.Json.Formatting.None,
new Newtonsoft.Json.JsonSerializerSettings()
{
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore
}), storeScope, true);
}
int icount = dynJson2.items is Newtonsoft.Json.Linq.JArray ? ((Newtonsoft.Json.Linq.JArray)dynJson2.items).Count : ((Newtonsoft.Json.Linq.JObject)dynJson2.items).Count;
//check if we have done with that category
if (dynJson2 == null || icount < 1 || icount < limit)
{
if (pt.MaxPrice > 0)
{
var cat = _categoryService.GetCategoryById(rcategory);
if (pt.MaxPrice > 50000)
{
cat.PriceRanges = "0-9999;10000-24999;25000-49999;50000-";
}
else if (pt.MaxPrice > 20000)
{
cat.PriceRanges = "0-4999;5000-9999;10000-24999;25000-";
}
else if (pt.MaxPrice > 5000)
{
cat.PriceRanges = "0-4999;5000-9999;10000-";
}
else if (pt.MaxPrice > 1000)
{
cat.PriceRanges = "0-999;1000-3999;4000-";
}
else
{
cat.PriceRanges = "0-249;250-499;500-749;750-";
}
_categoryService.UpdateCategory(cat);
}
//remove category that we have finished
categories.Remove(rcategory);
// store offset and everything for the next run (I was re-runing it by an ajax calls)
pt.Offset = 0;
pt.MaxPrice = 0;
pt.TotalProducts = 0;
pt.ProcessedProducts = 0;
_settingService.SetSetting<string>("DDProdcts", Newtonsoft.Json.JsonConvert.SerializeObject(pt, Newtonsoft.Json.Formatting.None,
new Newtonsoft.Json.JsonSerializerSettings()
{
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore
}), storeScope, true);
progress = string.Format("Categories: {0}, Products: {1}/{2} 'tbc", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
}
else // seems there are more products so we will continue on the same category
{
_settingService.SetSetting<string>("DDProdcts", Newtonsoft.Json.JsonConvert.SerializeObject(pt, Newtonsoft.Json.Formatting.None,
new Newtonsoft.Json.JsonSerializerSettings()
{
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore
}), storeScope, true);
progress = string.Format("Categories: {0}, Products: {1}/{2} 'tbc", ccc.Name + "/" + categories.Count, (string)dynJson2.count, prodProcessed);
}
}
catch (System.Net.WebException wex)
{
//LogException
}
catch (Exception ex)
{
//LogException
}
}
_cache.Set<string>("Progress", progress, TimeSpan.FromMinutes(30));
//seems we have more categories to work on
if (categories != null && categories.Count > 0)
{
_settingService.SetSetting<string>("DDCategories", Newtonsoft.Json.JsonConvert.SerializeObject(categories, Newtonsoft.Json.Formatting.None,
new Newtonsoft.Json.JsonSerializerSettings()
{
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore
}), storeScope, true);
}
else //seems we have done with all categories
{
progress = "Done";
_settingService.SetSetting<string>("DDProdcts", string.Empty, storeScope, true);
_settingService.SetSetting<string>("DDCategories", string.Empty, storeScope, true);
}
}
}
catch(Exception ex)
{
//LogException
}
return Json(new { data = progress });
}and the add pictures method
private void AddPictures(dynamic product, Core.Domain.Catalog.Product pro)
{
List<Tuple<string, byte[]>> bdata = new List<Tuple<string, byte[]>>();
var iurls = ((Newtonsoft.Json.Linq.JArray)product.imageFiles).Select(s => (string)s).ToList();
System.Threading.Tasks.Parallel.ForEach(iurls, iu =>
{
try
{
if (!bdata.Any(a => a.Item1 == iu))
{
System.Net.WebClient webClient = new System.Net.WebClient();
bdata.Add(new Tuple<string, byte[]>(iu, webClient.DownloadData(iu)));
}
}
catch (Exception ex)
{
//LogException
}
}
);
foreach (var iu in iurls)
{
try
{
var data = bdata.SingleOrDefault(s => s.Item1 == iu);
if (data != null)
{
string iufn = _fileProvider.GetFileName(iu);
string iufnl = iufn.ToLowerInvariant();
{
{
{
_pictureService.StoreInDb = false;
Core.Domain.Media.Picture propic = _pictureService.InsertPicture(data.Item2, "image/jpeg", iufn, pro.Name, pro.Name, true);
if (propic != null)
{
pro.ProductPictures.Add(new Core.Domain.Catalog.ProductPicture()
{
Picture = propic
});
}
}
}
}
}
}
catch (Exception ex)
{
//LogException
}
}
bdata.Clear();
}
This is not really 100% optimized, especially the DB adding/updating part because of using _services instead of _repositories
but seem I had problems using _repos everywhere and not sure why.
Thank you and see you soon
1vqHSTrq1GEoEF7QsL8dhmJfRMDVxhv2y