using AngleSharp.Html.Parser; using DTO; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; using zhengcaioa.Models; namespace Crawler.sichuan { public class sichuanoperation { public static string sichuanpageurll = "https://zfcg.scsczt.cn"; public static void operations(WebCrawlerContext _ccontext, zhengcaioaContext _ccontext1) { try { //获取更新时间 string operationStartTime = ""; Updatetime updatetime = _ccontext.Updatetimes.Where(x=>x.Sheng=="四川省").FirstOrDefault(); if (updatetime != null) { operationStartTime = updatetime.Updatetime1.ToString("yyyy-MM-dd")+ "%2000:00:00"; //updatetime.Updatetime1 = DateTime.Now.AddDays(-1); //_ccontext.SaveChanges(); } //else //{ // updatetime = new Updatetime(); // updatetime.Id = Guid.NewGuid().ToString(); // updatetime.Sheng = "四川省"; // updatetime.Updatetime1 = DateTime.Now.AddDays(-1); // _ccontext.Updatetimes.Add(updatetime); // _ccontext.SaveChanges(); //} //翻页信息 int currPage = 1; int pageSize = 10; //开始访问采购公告 logg.WriteLog("四川省意向公开开始获取", "四川"); while (true) { logg.WriteLog(currPage.ToString(), "四川"); string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=59®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea="; using (HttpClient client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage response = client.GetAsync(address).Result; var res = response.Content.ReadAsStringAsync().Result; sichuanjieshou sichuan = JsonConvert.DeserializeObject(res); //保存项目到sqlserveer和elasticsearch if (sichuan.data != null && sichuan.data.Count > 0) { foreach (var sichuanjieshoudtl in sichuan.data) { var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl; var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd"); var notimestart = DateTime.Parse(notime); var notimeend = notimestart.AddDays(1); var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "01" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); if (zhengfuProjectCount <= 0) { var zhengfuProject = new ZhengfuProject(); zhengfuProject.Id = Guid.NewGuid().ToString(); zhengfuProject.Gglx = "01"; string gglxName = "意向公开"; zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime; zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime; zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode; string regionName = sichuanjieshoudtl.regionName; zhengfuProject.Sheng = "510000"; string ShengName = "四川省"; string CityName = ""; if ("510001" == sichuanjieshoudtl.regionCode) { zhengfuProject.RegionCode = ""; regionName = ""; zhengfuProject.City = "510001"; CityName = "省本级"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5101")) { zhengfuProject.City = "510100"; CityName = "成都市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5103")) { zhengfuProject.City = "510300"; CityName = "自贡市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5104")) { zhengfuProject.City = "510400"; CityName = "攀枝花市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5105")) { zhengfuProject.City = "510500"; CityName = "泸州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5106")) { zhengfuProject.City = "510600"; CityName = "德阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5107")) { zhengfuProject.City = "510700"; CityName = "绵阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5108")) { zhengfuProject.City = "510800"; CityName = "广元市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5109")) { zhengfuProject.City = "510900"; CityName = "遂宁市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5110")) { zhengfuProject.City = "511000"; CityName = "内江市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5111")) { zhengfuProject.City = "511100"; CityName = "乐山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5113")) { zhengfuProject.City = "511300"; CityName = "南充市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5114")) { zhengfuProject.City = "5114"; CityName = "眉山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5115")) { zhengfuProject.City = "511500"; CityName = "宜宾市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5116")) { zhengfuProject.City = "511600"; CityName = "广安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5117")) { zhengfuProject.City = "511700"; CityName = "达州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5118")) { zhengfuProject.City = "511800"; CityName = "雅安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5119")) { zhengfuProject.City = "511900"; CityName = "巴中市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5120")) { zhengfuProject.City = "512000"; CityName = "资阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5132")) { zhengfuProject.City = "513200"; CityName = "阿坝州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5133")) { zhengfuProject.City = "513300"; CityName = "甘孜州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5134")) { zhengfuProject.City = "513400"; CityName = "凉山州"; } string cgfsName = ""; switch (sichuanjieshoudtl.purchaseManner) { case "1": zhengfuProject.Cgfs = "1"; cgfsName = "公开招标"; break; case "2": zhengfuProject.Cgfs = "2"; cgfsName = "邀请招标"; break; case "3": zhengfuProject.Cgfs = "5"; cgfsName = "竞争性谈判"; break; case "4": zhengfuProject.Cgfs = "4"; cgfsName = "询价"; break; case "5": zhengfuProject.Cgfs = "7"; cgfsName = "单一来源"; break; case "6": zhengfuProject.Cgfs = "3"; cgfsName = "竞争性磋商"; break; case "9": zhengfuProject.Cgfs = "10"; cgfsName = "其他"; break; default: zhengfuProject.Cgfs = "10"; cgfsName = "其他"; break; } zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode; zhengfuProject.Title = sichuanjieshoudtl.title; zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle; zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl; zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser; if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget)) { zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget); } zhengfuProject.Agency = sichuanjieshoudtl.agency; zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode; zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty); zhengfuProject.RecStatus = "A"; zhengfuProject.Creater = "1"; zhengfuProject.Createtime = DateTime.Now; zhengfuProject.Modifier = "1"; zhengfuProject.Modifytime = zhengfuProject.Createtime; _ccontext.ZhengfuProjects.Add(zhengfuProject); string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; string result = string.Empty; Uri postUrl = new Uri(url); eswebcrawler eswebcrawler1 = new eswebcrawler(); eswebcrawler1.Id = zhengfuProject.Id; eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; eswebcrawler1.sheng = zhengfuProject.Sheng; eswebcrawler1.shengName = ShengName; eswebcrawler1.city = zhengfuProject.City; eswebcrawler1.cityName = CityName; eswebcrawler1.regionCode = zhengfuProject.RegionCode; eswebcrawler1.regionName = regionName; eswebcrawler1.cgfs = zhengfuProject.Cgfs; eswebcrawler1.cgfsName = cgfsName; eswebcrawler1.gglx = zhengfuProject.Gglx; eswebcrawler1.gglxName = gglxName; eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; eswebcrawler1.title = zhengfuProject.Title; eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; eswebcrawler1.pageurl = zhengfuProject.Pageurl; eswebcrawler1.pingmu = zhengfuProject.Pingmu; eswebcrawler1.pingmuName = ""; eswebcrawler1.purchaser = zhengfuProject.Purchaser; eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; eswebcrawler1.agency = zhengfuProject.Agency; eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); string requestJson = JsonConvert.SerializeObject(eswebcrawler1); using (HttpContent httpContent = new StringContent(requestJson)) { httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); using (HttpClient httpClient = new HttpClient()) { httpClient.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; result = responseMessage.Content.ReadAsStringAsync().Result; } } Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") { } else { throw new Exception("保存数据库出错！"); } _ccontext.SaveChanges(); } } } if (currPage * pageSize >= sichuan.total) { break; } else { currPage += 1; } Thread.CurrentThread.Join(1000 * 20);//阻止设定时间 } } logg.WriteLog("四川省意向公开获取完成", "四川"); //翻页信息 currPage = 1; pageSize = 10; //开始访问采购公告 logg.WriteLog("四川省采购公告开始获取", "四川"); while (true) { logg.WriteLog(currPage.ToString(), "四川"); string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=00101®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea="; using (HttpClient client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage response = client.GetAsync(address).Result; var res = response.Content.ReadAsStringAsync().Result; sichuanjieshou sichuan = JsonConvert.DeserializeObject(res); //保存项目到sqlserveer和elasticsearch if (sichuan.data != null && sichuan.data.Count > 0) { foreach (var sichuanjieshoudtl in sichuan.data) { var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl; var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd"); var notimestart = DateTime.Parse(notime); var notimeend = notimestart.AddDays(1); var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "02" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); if (zhengfuProjectCount <= 0) { var zhengfuProject = new ZhengfuProject(); zhengfuProject.Id = Guid.NewGuid().ToString(); zhengfuProject.Gglx = "02"; string gglxName = "采购公告"; zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime; zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime; zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode; string regionName = sichuanjieshoudtl.regionName; zhengfuProject.Sheng = "510000"; string ShengName = "四川省"; string CityName = ""; if ("510001" == sichuanjieshoudtl.regionCode) { zhengfuProject.RegionCode = ""; regionName = ""; zhengfuProject.City = "510001"; CityName = "省本级"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5101")) { zhengfuProject.City = "510100"; CityName = "成都市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5103")) { zhengfuProject.City = "510300"; CityName = "自贡市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5104")) { zhengfuProject.City = "510400"; CityName = "攀枝花市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5105")) { zhengfuProject.City = "510500"; CityName = "泸州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5106")) { zhengfuProject.City = "510600"; CityName = "德阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5107")) { zhengfuProject.City = "510700"; CityName = "绵阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5108")) { zhengfuProject.City = "510800"; CityName = "广元市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5109")) { zhengfuProject.City = "510900"; CityName = "遂宁市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5110")) { zhengfuProject.City = "511000"; CityName = "内江市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5111")) { zhengfuProject.City = "511100"; CityName = "乐山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5113")) { zhengfuProject.City = "511300"; CityName = "南充市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5114")) { zhengfuProject.City = "5114"; CityName = "眉山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5115")) { zhengfuProject.City = "511500"; CityName = "宜宾市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5116")) { zhengfuProject.City = "511600"; CityName = "广安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5117")) { zhengfuProject.City = "511700"; CityName = "达州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5118")) { zhengfuProject.City = "511800"; CityName = "雅安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5119")) { zhengfuProject.City = "511900"; CityName = "巴中市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5120")) { zhengfuProject.City = "512000"; CityName = "资阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5132")) { zhengfuProject.City = "513200"; CityName = "阿坝州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5133")) { zhengfuProject.City = "513300"; CityName = "甘孜州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5134")) { zhengfuProject.City = "513400"; CityName = "凉山州"; } string cgfsName = ""; switch (sichuanjieshoudtl.purchaseManner) { case "1": zhengfuProject.Cgfs = "1"; cgfsName = "公开招标"; break; case "2": zhengfuProject.Cgfs = "2"; cgfsName = "邀请招标"; break; case "3": zhengfuProject.Cgfs = "5"; cgfsName = "竞争性谈判"; break; case "4": zhengfuProject.Cgfs = "4"; cgfsName = "询价"; break; case "5": zhengfuProject.Cgfs = "7"; cgfsName = "单一来源"; break; case "6": zhengfuProject.Cgfs = "3"; cgfsName = "竞争性磋商"; break; case "9": zhengfuProject.Cgfs = "10"; cgfsName = "其他"; break; default: zhengfuProject.Cgfs = "10"; cgfsName = "其他"; break; } zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode; zhengfuProject.Title = sichuanjieshoudtl.title; zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle; zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl; zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser; if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget)) { zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget); } zhengfuProject.Agency = sichuanjieshoudtl.agency; zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode; zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty); zhengfuProject.RecStatus = "A"; zhengfuProject.Creater = "1"; zhengfuProject.Createtime = DateTime.Now; zhengfuProject.Modifier = "1"; zhengfuProject.Modifytime = zhengfuProject.Createtime; _ccontext.ZhengfuProjects.Add(zhengfuProject); string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; string result = string.Empty; Uri postUrl = new Uri(url); eswebcrawler eswebcrawler1 = new eswebcrawler(); eswebcrawler1.Id = zhengfuProject.Id; eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; eswebcrawler1.sheng = zhengfuProject.Sheng; eswebcrawler1.shengName = ShengName; eswebcrawler1.city = zhengfuProject.City; eswebcrawler1.cityName = CityName; eswebcrawler1.regionCode = zhengfuProject.RegionCode; eswebcrawler1.regionName = regionName; eswebcrawler1.cgfs = zhengfuProject.Cgfs; eswebcrawler1.cgfsName = cgfsName; eswebcrawler1.gglx = zhengfuProject.Gglx; eswebcrawler1.gglxName = gglxName; eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; eswebcrawler1.title = zhengfuProject.Title; eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; eswebcrawler1.pageurl = zhengfuProject.Pageurl; eswebcrawler1.pingmu = zhengfuProject.Pingmu; eswebcrawler1.pingmuName = ""; eswebcrawler1.purchaser = zhengfuProject.Purchaser; eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; eswebcrawler1.agency = zhengfuProject.Agency; eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); string requestJson = JsonConvert.SerializeObject(eswebcrawler1); using (HttpContent httpContent = new StringContent(requestJson)) { httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); using (HttpClient httpClient = new HttpClient()) { httpClient.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; result = responseMessage.Content.ReadAsStringAsync().Result; } } Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") { } else { throw new Exception("保存数据库出错！"); } _ccontext.SaveChanges(); } } } if (currPage * pageSize >= sichuan.total) { break; } else { currPage += 1; } Thread.CurrentThread.Join(1000 * 20);//阻止设定时间 } } logg.WriteLog("四川省采购公告获取完成", "四川"); //翻页信息 currPage = 1; pageSize = 10; //开始访问采购公告 logg.WriteLog("四川省中标公告开始获取", "四川"); while (true) { logg.WriteLog(currPage.ToString(), "四川"); string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=00102®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea="; using (HttpClient client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage response = client.GetAsync(address).Result; var res = response.Content.ReadAsStringAsync().Result; sichuanjieshou sichuan = JsonConvert.DeserializeObject(res); //保存项目到sqlserveer和elasticsearch if (sichuan.data != null && sichuan.data.Count > 0) { foreach (var sichuanjieshoudtl in sichuan.data) { var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl; var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd"); var notimestart = DateTime.Parse(notime); var notimeend = notimestart.AddDays(1); var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "03" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); if (zhengfuProjectCount <= 0) { var zhengfuProject = new ZhengfuProject(); zhengfuProject.Id = Guid.NewGuid().ToString(); zhengfuProject.Gglx = "03"; string gglxName = "结果公告"; zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime; zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime; zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode; string regionName = sichuanjieshoudtl.regionName; zhengfuProject.Sheng = "510000"; string ShengName = "四川省"; string CityName = ""; if ("510001" == sichuanjieshoudtl.regionCode) { zhengfuProject.RegionCode = ""; regionName = ""; zhengfuProject.City = "510001"; CityName = "省本级"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5101")) { zhengfuProject.City = "510100"; CityName = "成都市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5103")) { zhengfuProject.City = "510300"; CityName = "自贡市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5104")) { zhengfuProject.City = "510400"; CityName = "攀枝花市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5105")) { zhengfuProject.City = "510500"; CityName = "泸州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5106")) { zhengfuProject.City = "510600"; CityName = "德阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5107")) { zhengfuProject.City = "510700"; CityName = "绵阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5108")) { zhengfuProject.City = "510800"; CityName = "广元市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5109")) { zhengfuProject.City = "510900"; CityName = "遂宁市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5110")) { zhengfuProject.City = "511000"; CityName = "内江市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5111")) { zhengfuProject.City = "511100"; CityName = "乐山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5113")) { zhengfuProject.City = "511300"; CityName = "南充市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5114")) { zhengfuProject.City = "511400"; CityName = "眉山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5115")) { zhengfuProject.City = "511500"; CityName = "宜宾市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5116")) { zhengfuProject.City = "511600"; CityName = "广安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5117")) { zhengfuProject.City = "511700"; CityName = "达州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5118")) { zhengfuProject.City = "511800"; CityName = "雅安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5119")) { zhengfuProject.City = "511900"; CityName = "巴中市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5120")) { zhengfuProject.City = "512000"; CityName = "资阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5132")) { zhengfuProject.City = "513200"; CityName = "阿坝州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5133")) { zhengfuProject.City = "513300"; CityName = "甘孜州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5134")) { zhengfuProject.City = "513400"; CityName = "凉山州"; } string cgfsName = ""; string BieCgfs = ""; switch (sichuanjieshoudtl.purchaseManner) { case "1": zhengfuProject.Cgfs = "1"; cgfsName = "公开招标"; BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c11"; break; case "2": zhengfuProject.Cgfs = "2"; cgfsName = "邀请招标"; BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c12"; break; case "3": zhengfuProject.Cgfs = "5"; cgfsName = "竞争性谈判"; BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c14"; break; case "4": zhengfuProject.Cgfs = "4"; cgfsName = "询价"; BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c15"; break; case "5": zhengfuProject.Cgfs = "7"; cgfsName = "单一来源"; BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c16"; break; case "6": zhengfuProject.Cgfs = "3"; cgfsName = "竞争性磋商"; BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c13"; break; case "9": zhengfuProject.Cgfs = "10"; cgfsName = "其他"; BieCgfs = ""; break; default: zhengfuProject.Cgfs = "10"; cgfsName = "其他"; BieCgfs = ""; break; } zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode; zhengfuProject.Title = sichuanjieshoudtl.title; zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle; zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl; zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser; if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget)) { zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget); } zhengfuProject.Agency = sichuanjieshoudtl.agency; zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode; zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty); zhengfuProject.RecStatus = "A"; zhengfuProject.Creater = "1"; zhengfuProject.Createtime = DateTime.Now; zhengfuProject.Modifier = "1"; zhengfuProject.Modifytime = zhengfuProject.Createtime; _ccontext.ZhengfuProjects.Add(zhengfuProject); Project project = new Project(); project.Id = zhengfuProject.Id; project.StartTime = zhengfuProject.NoticeTime; project.Sheng = zhengfuProject.Sheng; project.City = zhengfuProject.City; if (project.City == "510001") { project.AreaId = "510002"; } else { project.AreaId = zhengfuProject.RegionCode; } project.Cgfs = BieCgfs; project.Number = zhengfuProject.OpenTenderCode; project.Name = zhengfuProject.Title; if (zhengfuProject.Budget.HasValue) { project.Caigouyusuan = zhengfuProject.Budget.Value / new decimal(10000); } project.Question = zhengfuProject.Pageurl; var parser = new HtmlParser(); var document = parser.ParseDocument(zhengfuProject.Content); var cDocumet = document.All.Where(m => m.Id == "noticeArea").FirstOrDefault(); if (cDocumet != null) { //var cDocumet = contentList.First(); var Numberss = cDocumet.QuerySelectorAll("p"); foreach (var Number in Numberss) { if (Number != null && Number.TextContent.Contains("废标理由")) { var Fbyy = Number.QuerySelector("span"); project.Fbyy = Fbyy.TextContent.Trim(); } } var Caigourennames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgName"); if (Caigourennames != null && Caigourennames.ToList().Count > 0) { var Caigourenname = Caigourennames.First(); project.Caigourenname = Caigourenname.TextContent; } var Caigourenaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgAddress"); if (Caigourenaddresss != null && Caigourenaddresss.ToList().Count > 0) { var Caigourenaddress = Caigourenaddresss.First(); project.Caigourenaddress = Caigourenaddress.TextContent; } var Caigourencontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserLinkTel"); if (Caigourencontacts != null && Caigourencontacts.ToList().Count > 0) { var Caigourencontact = Caigourencontacts.First(); project.Caigourencontact = Caigourencontact.TextContent; } var Dailinames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agencyName"); if (Dailinames != null && Dailinames.ToList().Count > 0) { var Dailiname = Dailinames.First(); project.Dailiname = Dailiname.TextContent; } var Dailiaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentAddress"); if (Dailiaddresss != null && Dailiaddresss.ToList().Count > 0) { var Dailiaddress = Dailiaddresss.First(); project.Dailiaddress = Dailiaddress.TextContent; } var Dailicontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentLinkTel"); if (Dailicontacts != null && Dailicontacts.ToList().Count > 0) { var Dailicontact = Dailicontacts.First(); project.Dailicontact = Dailicontact.TextContent; } var zhuanjias = cDocumet.QuerySelectorAll("div").Where(m => m.ClassName == "noticeBidResult-noticeBidResult _notice_content_noticeBidResult-noticeBidResult expert dynamic-form-editor"); if (zhuanjias != null && zhuanjias.ToList().Count > 0) { var zhuanjia = zhuanjias.First(); project.Zhuanjia = zhuanjia.TextContent.Trim();//OuterHtml } } project.Creater = "111"; project.Createtime = zhengfuProject.Createtime; project.Modifier = project.Creater; project.Modifytime = project.Createtime; project.RecStatus = "A"; _ccontext1.Projects.Add(project); if (cDocumet != null) { //var cDocumet = contentList.First(); var table1 = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_noticeBidResult-noticeBidResult"); if (table1 != null && table1.ToList().Count > 0) { var table = table1.First().QuerySelectorAll("table"); if (table != null && table.ToList().Count > 0) { for (int i = 0; i < table.Length; i++) { var td = table[i].QuerySelectorAll("td"); if (td.Length >= 3) { var projectzhongbiao = new Projectzhongbiao(); projectzhongbiao.ZhongbiaoId = Guid.NewGuid().ToString(); projectzhongbiao.ProjectId = project.Id; if ( !string.IsNullOrWhiteSpace(td[0].TextContent) && td[0].TextContent.Trim().Length > 500) { projectzhongbiao.Names = td[0].TextContent.Trim().Substring(0,500); } else { projectzhongbiao.Names = td[0].TextContent.Trim(); } projectzhongbiao.Price = td[2].TextContent.Trim(); _ccontext1.Projectzhongbiaos.Add(projectzhongbiao); } } } } } string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; string result = string.Empty; Uri postUrl = new Uri(url); eswebcrawler eswebcrawler1 = new eswebcrawler(); eswebcrawler1.Id = zhengfuProject.Id; eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; eswebcrawler1.sheng = zhengfuProject.Sheng; eswebcrawler1.shengName = ShengName; eswebcrawler1.city = zhengfuProject.City; eswebcrawler1.cityName = CityName; eswebcrawler1.regionCode = zhengfuProject.RegionCode; eswebcrawler1.regionName = regionName; eswebcrawler1.cgfs = zhengfuProject.Cgfs; eswebcrawler1.cgfsName = cgfsName; eswebcrawler1.gglx = zhengfuProject.Gglx; eswebcrawler1.gglxName = gglxName; eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; eswebcrawler1.title = zhengfuProject.Title; eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; eswebcrawler1.pageurl = zhengfuProject.Pageurl; eswebcrawler1.pingmu = zhengfuProject.Pingmu; eswebcrawler1.pingmuName = ""; eswebcrawler1.purchaser = zhengfuProject.Purchaser; eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; eswebcrawler1.agency = zhengfuProject.Agency; eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); string requestJson = JsonConvert.SerializeObject(eswebcrawler1); using (HttpContent httpContent = new StringContent(requestJson)) { httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); using (HttpClient httpClient = new HttpClient()) { httpClient.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; result = responseMessage.Content.ReadAsStringAsync().Result; } } Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") { } else { throw new Exception("保存数据库出错！"); } _ccontext1.SaveChanges(); _ccontext.SaveChanges(); } } } if (currPage * pageSize >= sichuan.total) { break; } else { currPage += 1; } Thread.CurrentThread.Join(1000 * 20);//阻止设定时间 } } logg.WriteLog("四川省中标公告获取完成", "四川"); //翻页信息 currPage = 1; pageSize = 10; //开始访问采购公告 logg.WriteLog("四川省更正公告开始获取", "四川"); while (true) { logg.WriteLog(currPage.ToString(), "四川"); string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=00103®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea="; using (HttpClient client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage response = client.GetAsync(address).Result; var res = response.Content.ReadAsStringAsync().Result; sichuanjieshou sichuan = JsonConvert.DeserializeObject(res); //保存项目到sqlserveer和elasticsearch if (sichuan.data != null && sichuan.data.Count > 0) { foreach (var sichuanjieshoudtl in sichuan.data) { var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl; var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd"); var notimestart = DateTime.Parse(notime); var notimeend = notimestart.AddDays(1); var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "04" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); if (zhengfuProjectCount <= 0) { var zhengfuProject = new ZhengfuProject(); zhengfuProject.Id = Guid.NewGuid().ToString(); zhengfuProject.Gglx = "04"; string gglxName = "更正公告"; zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime; zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime; zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode; string regionName = sichuanjieshoudtl.regionName; zhengfuProject.Sheng = "510000"; string ShengName = "四川省"; string CityName = ""; if ("510001" == sichuanjieshoudtl.regionCode) { zhengfuProject.RegionCode = ""; regionName = ""; zhengfuProject.City = "510001"; CityName = "省本级"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5101")) { zhengfuProject.City = "510100"; CityName = "成都市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5103")) { zhengfuProject.City = "510300"; CityName = "自贡市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5104")) { zhengfuProject.City = "510400"; CityName = "攀枝花市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5105")) { zhengfuProject.City = "510500"; CityName = "泸州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5106")) { zhengfuProject.City = "510600"; CityName = "德阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5107")) { zhengfuProject.City = "510700"; CityName = "绵阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5108")) { zhengfuProject.City = "510800"; CityName = "广元市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5109")) { zhengfuProject.City = "510900"; CityName = "遂宁市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5110")) { zhengfuProject.City = "511000"; CityName = "内江市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5111")) { zhengfuProject.City = "511100"; CityName = "乐山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5113")) { zhengfuProject.City = "511300"; CityName = "南充市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5114")) { zhengfuProject.City = "5114"; CityName = "眉山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5115")) { zhengfuProject.City = "511500"; CityName = "宜宾市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5116")) { zhengfuProject.City = "511600"; CityName = "广安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5117")) { zhengfuProject.City = "511700"; CityName = "达州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5118")) { zhengfuProject.City = "511800"; CityName = "雅安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5119")) { zhengfuProject.City = "511900"; CityName = "巴中市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5120")) { zhengfuProject.City = "512000"; CityName = "资阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5132")) { zhengfuProject.City = "513200"; CityName = "阿坝州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5133")) { zhengfuProject.City = "513300"; CityName = "甘孜州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5134")) { zhengfuProject.City = "513400"; CityName = "凉山州"; } string cgfsName = ""; switch (sichuanjieshoudtl.purchaseManner) { case "1": zhengfuProject.Cgfs = "1"; cgfsName = "公开招标"; break; case "2": zhengfuProject.Cgfs = "2"; cgfsName = "邀请招标"; break; case "3": zhengfuProject.Cgfs = "5"; cgfsName = "竞争性谈判"; break; case "4": zhengfuProject.Cgfs = "4"; cgfsName = "询价"; break; case "5": zhengfuProject.Cgfs = "7"; cgfsName = "单一来源"; break; case "6": zhengfuProject.Cgfs = "3"; cgfsName = "竞争性磋商"; break; case "9": zhengfuProject.Cgfs = "10"; cgfsName = "其他"; break; default: zhengfuProject.Cgfs = "10"; cgfsName = "其他"; break; } zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode; zhengfuProject.Title = sichuanjieshoudtl.title; zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle; zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl; zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser; if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget)) { zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget); } zhengfuProject.Agency = sichuanjieshoudtl.agency; zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode; zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty); zhengfuProject.RecStatus = "A"; zhengfuProject.Creater = "1"; zhengfuProject.Createtime = DateTime.Now; zhengfuProject.Modifier = "1"; zhengfuProject.Modifytime = zhengfuProject.Createtime; _ccontext.ZhengfuProjects.Add(zhengfuProject); string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; string result = string.Empty; Uri postUrl = new Uri(url); eswebcrawler eswebcrawler1 = new eswebcrawler(); eswebcrawler1.Id = zhengfuProject.Id; eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; eswebcrawler1.sheng = zhengfuProject.Sheng; eswebcrawler1.shengName = ShengName; eswebcrawler1.city = zhengfuProject.City; eswebcrawler1.cityName = CityName; eswebcrawler1.regionCode = zhengfuProject.RegionCode; eswebcrawler1.regionName = regionName; eswebcrawler1.cgfs = zhengfuProject.Cgfs; eswebcrawler1.cgfsName = cgfsName; eswebcrawler1.gglx = zhengfuProject.Gglx; eswebcrawler1.gglxName = gglxName; eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; eswebcrawler1.title = zhengfuProject.Title; eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; eswebcrawler1.pageurl = zhengfuProject.Pageurl; eswebcrawler1.pingmu = zhengfuProject.Pingmu; eswebcrawler1.pingmuName = ""; eswebcrawler1.purchaser = zhengfuProject.Purchaser; eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; eswebcrawler1.agency = zhengfuProject.Agency; eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); string requestJson = JsonConvert.SerializeObject(eswebcrawler1); using (HttpContent httpContent = new StringContent(requestJson)) { httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); using (HttpClient httpClient = new HttpClient()) { httpClient.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; result = responseMessage.Content.ReadAsStringAsync().Result; } } Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") { } else { throw new Exception("保存数据库出错！"); } _ccontext.SaveChanges(); } } } if (currPage * pageSize >= sichuan.total) { break; } else { currPage += 1; } Thread.CurrentThread.Join(1000 * 20);//阻止设定时间 } } logg.WriteLog("四川省更正公告获取完成", "四川"); //翻页信息 currPage = 1; pageSize = 10; //开始访问采购公告 logg.WriteLog("四川省废标公告开始获取", "四川"); while (true) { logg.WriteLog(currPage.ToString(), "四川"); string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=001004,001006®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea="; using (HttpClient client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage response = client.GetAsync(address).Result; var res = response.Content.ReadAsStringAsync().Result; sichuanjieshou sichuan = JsonConvert.DeserializeObject(res); //保存项目到sqlserveer和elasticsearch if (sichuan.data != null && sichuan.data.Count > 0) { foreach (var sichuanjieshoudtl in sichuan.data) { var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl; var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd"); var notimestart = DateTime.Parse(notime); var notimeend = notimestart.AddDays(1); var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "11" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend)); if (zhengfuProjectCount <= 0) { var zhengfuProject = new ZhengfuProject(); zhengfuProject.Id = Guid.NewGuid().ToString(); zhengfuProject.Gglx = "11"; string gglxName = "废标公告"; zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime; zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime; zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode; string regionName = sichuanjieshoudtl.regionName; zhengfuProject.Sheng = "510000"; string ShengName = "四川省"; string CityName = ""; if ("510001" == sichuanjieshoudtl.regionCode) { zhengfuProject.RegionCode = ""; regionName = ""; zhengfuProject.City = "510001"; CityName = "省本级"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5101")) { zhengfuProject.City = "510100"; CityName = "成都市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5103")) { zhengfuProject.City = "510300"; CityName = "自贡市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5104")) { zhengfuProject.City = "510400"; CityName = "攀枝花市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5105")) { zhengfuProject.City = "510500"; CityName = "泸州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5106")) { zhengfuProject.City = "510600"; CityName = "德阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5107")) { zhengfuProject.City = "510700"; CityName = "绵阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5108")) { zhengfuProject.City = "510800"; CityName = "广元市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5109")) { zhengfuProject.City = "510900"; CityName = "遂宁市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5110")) { zhengfuProject.City = "511000"; CityName = "内江市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5111")) { zhengfuProject.City = "511100"; CityName = "乐山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5113")) { zhengfuProject.City = "511300"; CityName = "南充市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5114")) { zhengfuProject.City = "5114"; CityName = "眉山市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5115")) { zhengfuProject.City = "511500"; CityName = "宜宾市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5116")) { zhengfuProject.City = "511600"; CityName = "广安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5117")) { zhengfuProject.City = "511700"; CityName = "达州市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5118")) { zhengfuProject.City = "511800"; CityName = "雅安市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5119")) { zhengfuProject.City = "511900"; CityName = "巴中市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5120")) { zhengfuProject.City = "512000"; CityName = "资阳市"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5132")) { zhengfuProject.City = "513200"; CityName = "阿坝州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5133")) { zhengfuProject.City = "513300"; CityName = "甘孜州"; } else if (sichuanjieshoudtl.regionCode.StartsWith("5134")) { zhengfuProject.City = "513400"; CityName = "凉山州"; } string cgfsName = ""; switch (sichuanjieshoudtl.purchaseManner) { case "1": zhengfuProject.Cgfs = "1"; cgfsName = "公开招标"; break; case "2": zhengfuProject.Cgfs = "2"; cgfsName = "邀请招标"; break; case "3": zhengfuProject.Cgfs = "5"; cgfsName = "竞争性谈判"; break; case "4": zhengfuProject.Cgfs = "4"; cgfsName = "询价"; break; case "5": zhengfuProject.Cgfs = "7"; cgfsName = "单一来源"; break; case "6": zhengfuProject.Cgfs = "3"; cgfsName = "竞争性磋商"; break; case "9": zhengfuProject.Cgfs = "10"; cgfsName = "其他"; break; default: zhengfuProject.Cgfs = "10"; cgfsName = "其他"; break; } zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode; zhengfuProject.Title = sichuanjieshoudtl.title; zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle; zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl; zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser; if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget)) { zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget); } zhengfuProject.Agency = sichuanjieshoudtl.agency; zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode; zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty); zhengfuProject.RecStatus = "A"; zhengfuProject.Creater = "1"; zhengfuProject.Createtime = DateTime.Now; zhengfuProject.Modifier = "1"; zhengfuProject.Modifytime = zhengfuProject.Createtime; _ccontext.ZhengfuProjects.Add(zhengfuProject); var projects = _ccontext1.Projects.Where(x => x.RecStatus == "A" && x.Number == zhengfuProject.OpenTenderCode && x.Fblx != null ).ToList(); if (projects.Count > 0) { foreach (var project in projects) { project.Fblx = "停止评审"; } var parser = new HtmlParser(); var document = parser.ParseDocument(zhengfuProject.Content); var contentList = document.All.Where(m => m.Id == "noticeArea"); var cDocumet = contentList.First(); var Numberss = cDocumet.QuerySelectorAll("p"); foreach (var Number in Numberss) { if (Number != null && Number.TextContent.Contains("废标理由")) { if (Number.TextContent.IndexOf("span") >= 0) { var Fbyy = Number.QuerySelector("span"); foreach (var project in projects) { project.Fbyy = Fbyy.TextContent.Trim(); project.Fblx = "停止评审"; } } } if (Number != null && Number.TextContent.Contains("终止原因")) { if(Number.TextContent.IndexOf("span") >= 0) { var Fbyy = Number.QuerySelector("span"); foreach (var project in projects) { project.Fbyy = Fbyy.TextContent.Trim(); project.Fblx = "停止评审"; } } } } var feibiaoliyous = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_reason-reason"); if (feibiaoliyous != null && feibiaoliyous.ToList().Count > 0) { var feibiaoliyou = feibiaoliyous.First(); foreach (var project in projects) { project.Fbyy = feibiaoliyou.TextContent.Replace("\n", "").Replace("\t", ""); project.Fblx = "终止采购"; } } _ccontext1.SaveChanges(); } string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id; string result = string.Empty; Uri postUrl = new Uri(url); eswebcrawler eswebcrawler1 = new eswebcrawler(); eswebcrawler1.Id = zhengfuProject.Id; eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss"); eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null; eswebcrawler1.sheng = zhengfuProject.Sheng; eswebcrawler1.shengName = ShengName; eswebcrawler1.city = zhengfuProject.City; eswebcrawler1.cityName = CityName; eswebcrawler1.regionCode = zhengfuProject.RegionCode; eswebcrawler1.regionName = regionName; eswebcrawler1.cgfs = zhengfuProject.Cgfs; eswebcrawler1.cgfsName = cgfsName; eswebcrawler1.gglx = zhengfuProject.Gglx; eswebcrawler1.gglxName = gglxName; eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode; eswebcrawler1.title = zhengfuProject.Title; eswebcrawler1.shorttitle = zhengfuProject.Shorttitle; eswebcrawler1.pageurl = zhengfuProject.Pageurl; eswebcrawler1.pingmu = zhengfuProject.Pingmu; eswebcrawler1.pingmuName = ""; eswebcrawler1.purchaser = zhengfuProject.Purchaser; eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : ""; eswebcrawler1.agency = zhengfuProject.Agency; eswebcrawler1.agencyCode = zhengfuProject.AgencyCode; eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss"); string requestJson = JsonConvert.SerializeObject(eswebcrawler1); using (HttpContent httpContent = new StringContent(requestJson)) { httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); using (HttpClient httpClient = new HttpClient()) { httpClient.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result; result = responseMessage.Content.ReadAsStringAsync().Result; } } Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result); if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1") { } else { throw new Exception("保存数据库出错！"); } _ccontext.SaveChanges(); } } } if (currPage * pageSize >= sichuan.total) { break; } else { currPage += 1; } Thread.CurrentThread.Join(1000 * 20);//阻止设定时间 } } logg.WriteLog("四川省废标公告获取完成", "四川"); //保存最后更新时间 if (updatetime != null) { updatetime.Updatetime1 = DateTime.Now.AddDays(-1); _ccontext.SaveChanges(); } else { updatetime = new Updatetime(); updatetime.Id = Guid.NewGuid().ToString(); updatetime.Sheng = "四川省"; updatetime.Updatetime1 = DateTime.Now.AddDays(-1); _ccontext.Updatetimes.Add(updatetime); _ccontext.SaveChanges(); } } catch (Exception ex) { logg.WriteLog(ex.Message, "四川"); logg.WriteLog(ex.StackTrace, "四川"); logg.WriteLog(ex.ToString(), "四川"); } finally { } } } }