From 2d7c6f2bd3071d0d458765ae108c98c88669e412 Mon Sep 17 00:00:00 2001
From: username@email.com <yzy2002yzy@163.com>
Date: 星期六, 28 一月 2023 13:41:23 +0800
Subject: [PATCH] 提交
---
zhengcaioa/Crawler/sichuan/sichuanoperation.cs | 835 +++++++++++++++++++++++++++++++++--------------------------
1 files changed, 466 insertions(+), 369 deletions(-)
diff --git a/zhengcaioa/Crawler/sichuan/sichuanoperation.cs b/zhengcaioa/Crawler/sichuan/sichuanoperation.cs
index 5ce1381..bfcf20c 100644
--- a/zhengcaioa/Crawler/sichuan/sichuanoperation.cs
+++ b/zhengcaioa/Crawler/sichuan/sichuanoperation.cs
@@ -44,6 +44,294 @@
int currPage = 1;
int pageSize = 10;
//寮�濮嬭闂� 閲囪喘鍏憡
+ logg.WriteLog("鍥涘窛鐪佹剰鍚戝叕寮� 寮�濮嬭幏鍙�", "鍥涘窛");
+ while (true)
+ {
+ logg.WriteLog(currPage.ToString(), "鍥涘窛");
+ string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=59®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
+ using (HttpClient client = new HttpClient())
+ {
+ client.Timeout = TimeSpan.FromSeconds(60);
+ HttpResponseMessage response = client.GetAsync(address).Result;
+ var res = response.Content.ReadAsStringAsync().Result;
+
+ sichuanjieshou sichuan = JsonConvert.DeserializeObject<sichuanjieshou>(res);
+
+ //淇濆瓨椤圭洰鍒皊qlserveer鍜宔lasticsearch
+ if (sichuan.data != null && sichuan.data.Count > 0)
+ {
+ foreach (var sichuanjieshoudtl in sichuan.data)
+ {
+ var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
+ var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "01" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
+ if (zhengfuProjectCount <= 0)
+ {
+ var zhengfuProject = new ZhengfuProject();
+ zhengfuProject.Id = Guid.NewGuid().ToString();
+ zhengfuProject.Gglx = "01";
+ string gglxName = "鎰忓悜鍏紑";
+ zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime;
+ zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime;
+ zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode;
+ string regionName = sichuanjieshoudtl.regionName;
+ zhengfuProject.Sheng = "510000";
+ string ShengName = "鍥涘窛鐪�";
+ string CityName = "";
+ if ("510001" == sichuanjieshoudtl.regionCode)
+ {
+ zhengfuProject.RegionCode = "";
+ regionName = "";
+ zhengfuProject.City = "510001";
+ CityName = "鐪佹湰绾�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5101"))
+ {
+ zhengfuProject.City = "510100";
+ CityName = "鎴愰兘甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5103"))
+ {
+ zhengfuProject.City = "510300";
+ CityName = "鑷础甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5104"))
+ {
+ zhengfuProject.City = "510400";
+ CityName = "鏀�鏋濊姳甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5105"))
+ {
+ zhengfuProject.City = "510500";
+ CityName = "娉稿窞甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5106"))
+ {
+ zhengfuProject.City = "510600";
+ CityName = "寰烽槼甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5107"))
+ {
+ zhengfuProject.City = "510700";
+ CityName = "缁甸槼甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5108"))
+ {
+ zhengfuProject.City = "510800";
+ CityName = "骞垮厓甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5109"))
+ {
+ zhengfuProject.City = "510900";
+ CityName = "閬傚畞甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5110"))
+ {
+ zhengfuProject.City = "511000";
+ CityName = "鍐呮睙甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5111"))
+ {
+ zhengfuProject.City = "511100";
+ CityName = "涔愬北甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5113"))
+ {
+ zhengfuProject.City = "511300";
+ CityName = "鍗楀厖甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5114"))
+ {
+ zhengfuProject.City = "5114";
+ CityName = "鐪夊北甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5115"))
+ {
+ zhengfuProject.City = "511500";
+ CityName = "瀹滃甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5116"))
+ {
+ zhengfuProject.City = "511600";
+ CityName = "骞垮畨甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5117"))
+ {
+ zhengfuProject.City = "511700";
+ CityName = "杈惧窞甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5118"))
+ {
+ zhengfuProject.City = "511800";
+ CityName = "闆呭畨甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5119"))
+ {
+ zhengfuProject.City = "511900";
+ CityName = "宸翠腑甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5120"))
+ {
+ zhengfuProject.City = "512000";
+ CityName = "璧勯槼甯�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5132"))
+ {
+ zhengfuProject.City = "513200";
+ CityName = "闃垮潩宸�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5133"))
+ {
+ zhengfuProject.City = "513300";
+ CityName = "鐢樺瓬宸�";
+ }
+ else if (sichuanjieshoudtl.regionCode.StartsWith("5134"))
+ {
+ zhengfuProject.City = "513400";
+ CityName = "鍑夊北宸�";
+ }
+
+
+ string cgfsName = "";
+ switch (sichuanjieshoudtl.purchaseManner)
+ {
+ case "1":
+ zhengfuProject.Cgfs = "1";
+ cgfsName = "鍏紑鎷涙爣";
+ break;
+ case "2":
+ zhengfuProject.Cgfs = "2";
+ cgfsName = "閭�璇锋嫑鏍�";
+ break;
+ case "3":
+ zhengfuProject.Cgfs = "5";
+ cgfsName = "绔炰簤鎬ц皥鍒�";
+ break;
+ case "4":
+ zhengfuProject.Cgfs = "4";
+ cgfsName = "璇环";
+ break;
+ case "5":
+ zhengfuProject.Cgfs = "7";
+ cgfsName = "鍗曚竴鏉ユ簮";
+ break;
+ case "6":
+ zhengfuProject.Cgfs = "3";
+ cgfsName = "绔炰簤鎬х鍟�";
+ break;
+ case "9":
+ zhengfuProject.Cgfs = "10";
+ cgfsName = "鍏朵粬";
+ break;
+ default:
+ zhengfuProject.Cgfs = "10";
+ cgfsName = "鍏朵粬";
+ break;
+ }
+ zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
+ zhengfuProject.Title = sichuanjieshoudtl.title;
+ zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle;
+ zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl;
+ zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser;
+
+ if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget))
+ {
+ zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget);
+ }
+ zhengfuProject.Agency = sichuanjieshoudtl.agency;
+ zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode;
+ zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty);
+ zhengfuProject.RecStatus = "A";
+ zhengfuProject.Creater = "1";
+ zhengfuProject.Createtime = DateTime.Now;
+ zhengfuProject.Modifier = "1";
+ zhengfuProject.Modifytime = zhengfuProject.Createtime;
+ _ccontext.ZhengfuProjects.Add(zhengfuProject);
+
+
+ string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
+ string result = string.Empty;
+ Uri postUrl = new Uri(url);
+ eswebcrawler eswebcrawler1 = new eswebcrawler();
+ eswebcrawler1.Id = zhengfuProject.Id;
+ eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss");
+ eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null;
+ eswebcrawler1.sheng = zhengfuProject.Sheng;
+ eswebcrawler1.shengName = ShengName;
+ eswebcrawler1.city = zhengfuProject.City;
+ eswebcrawler1.cityName = CityName;
+ eswebcrawler1.regionCode = zhengfuProject.RegionCode;
+ eswebcrawler1.regionName = regionName;
+ eswebcrawler1.cgfs = zhengfuProject.Cgfs;
+ eswebcrawler1.cgfsName = cgfsName;
+ eswebcrawler1.gglx = zhengfuProject.Gglx;
+ eswebcrawler1.gglxName = gglxName;
+ eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode;
+ eswebcrawler1.title = zhengfuProject.Title;
+ eswebcrawler1.shorttitle = zhengfuProject.Shorttitle;
+ eswebcrawler1.pageurl = zhengfuProject.Pageurl;
+ eswebcrawler1.pingmu = zhengfuProject.Pingmu;
+ eswebcrawler1.pingmuName = "";
+ eswebcrawler1.purchaser = zhengfuProject.Purchaser;
+ eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : "";
+ eswebcrawler1.agency = zhengfuProject.Agency;
+ eswebcrawler1.agencyCode = zhengfuProject.AgencyCode;
+ eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss");
+
+ string requestJson = JsonConvert.SerializeObject(eswebcrawler1);
+
+ using (HttpContent httpContent = new StringContent(requestJson))
+ {
+ httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+ using (HttpClient httpClient = new HttpClient())
+ {
+ httpClient.Timeout = TimeSpan.FromSeconds(60);
+ HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result;
+ result = responseMessage.Content.ReadAsStringAsync().Result;
+ }
+
+ }
+
+ Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result);
+
+ if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1")
+ {
+
+ }
+ else
+ {
+ throw new Exception("淇濆瓨鏁版嵁搴撳嚭閿欙紒");
+ }
+
+
+ _ccontext.SaveChanges();
+ }
+ }
+ }
+
+
+ if (currPage * pageSize >= sichuan.total)
+ {
+ break;
+ }
+ else
+ {
+ currPage += 1;
+ }
+ Thread.CurrentThread.Join(1000 * 2);//闃绘璁惧畾鏃堕棿
+ }
+
+
+ }
+ logg.WriteLog("鍥涘窛鐪佹剰鍚戝叕寮� 鑾峰彇瀹屾垚", "鍥涘窛");
+
+ //缈婚〉淇℃伅
+ currPage = 1;
+ pageSize = 10;
+ //寮�濮嬭闂� 閲囪喘鍏憡
logg.WriteLog("鍥涘窛鐪侀噰璐叕鍛� 寮�濮嬭幏鍙�", "鍥涘窛");
while (true)
{
@@ -63,10 +351,13 @@
foreach (var sichuanjieshoudtl in sichuan.data)
{
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
- ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == pageurl1).FirstOrDefault();
- if (zhengfuProject == null)
+ var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "02" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
+ if (zhengfuProjectCount <= 0)
{
- zhengfuProject = new ZhengfuProject();
+ var zhengfuProject = new ZhengfuProject();
zhengfuProject.Id = Guid.NewGuid().ToString();
zhengfuProject.Gglx = "02";
string gglxName = "閲囪喘鍏憡";
@@ -322,7 +613,7 @@
}
- logg.WriteLog("鍥涘窛鐪侀噰璐叕鍛� 鑾峰彇瀹屾垚","鍥涘窛");
+ logg.WriteLog("鍥涘窛鐪侀噰璐叕鍛� 鑾峰彇瀹屾垚", "鍥涘窛");
//缈婚〉淇℃伅
currPage = 1;
@@ -347,10 +638,13 @@
foreach (var sichuanjieshoudtl in sichuan.data)
{
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
- ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == pageurl1).FirstOrDefault();
- if (zhengfuProject == null)
+ var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "03" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
+ if (zhengfuProjectCount <= 0)
{
- zhengfuProject = new ZhengfuProject();
+ var zhengfuProject = new ZhengfuProject();
zhengfuProject.Id = Guid.NewGuid().ToString();
zhengfuProject.Gglx = "03";
string gglxName = "缁撴灉鍏憡";
@@ -546,15 +840,15 @@
project.StartTime = zhengfuProject.NoticeTime;
project.Sheng = zhengfuProject.Sheng;
project.City = zhengfuProject.City;
- if(project.City== "510001")
+ if (project.City == "510001")
{
- project.AreaId = "510002";
+ project.AreaId = "510002";
}
else
{
project.AreaId = zhengfuProject.RegionCode;
}
-
+
project.Cgfs = BieCgfs;
project.Number = zhengfuProject.OpenTenderCode;
project.Name = zhengfuProject.Title;
@@ -566,64 +860,74 @@
project.Question = zhengfuProject.Pageurl;
var parser = new HtmlParser();
var document = parser.ParseDocument(zhengfuProject.Content);
- var contentList = document.All.Where(m => m.Id == "noticeArea");
- var cDocumet = contentList.First();
- var Numberss = cDocumet.QuerySelectorAll("p");
- foreach (var Number in Numberss)
+ var cDocumet = document.All.Where(m => m.Id == "noticeArea").FirstOrDefault();
+ if (cDocumet != null)
{
-
- if (Number != null && Number.TextContent.Contains("搴熸爣鐞嗙敱"))
+ //var cDocumet = contentList.First();
+ var Numberss = cDocumet.QuerySelectorAll("p");
+ foreach (var Number in Numberss)
{
- var Fbyy = Number.QuerySelector("span");
- project.Fbyy = Fbyy.TextContent.Trim();
+
+ if (Number != null && Number.TextContent.Contains("搴熸爣鐞嗙敱"))
+ {
+ var Fbyy = Number.QuerySelector("span");
+ project.Fbyy = Fbyy.TextContent.Trim();
+ }
+ }
+
+ var Caigourennames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgName");
+ if (Caigourennames != null && Caigourennames.ToList().Count > 0)
+ {
+ var Caigourenname = Caigourennames.First();
+ project.Caigourenname = Caigourenname.TextContent;
+ }
+ var Caigourenaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgAddress");
+ if (Caigourenaddresss != null && Caigourenaddresss.ToList().Count > 0)
+ {
+ var Caigourenaddress = Caigourenaddresss.First();
+ project.Caigourenaddress = Caigourenaddress.TextContent;
+ }
+ var Caigourencontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserLinkTel");
+ if (Caigourencontacts != null && Caigourencontacts.ToList().Count > 0)
+ {
+ var Caigourencontact = Caigourencontacts.First();
+ project.Caigourencontact = Caigourencontact.TextContent;
+ }
+ var Dailinames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agencyName");
+ if (Dailinames != null && Dailinames.ToList().Count > 0)
+ {
+ var Dailiname = Dailinames.First();
+ project.Dailiname = Dailiname.TextContent;
+ }
+ var Dailiaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentAddress");
+ if (Dailiaddresss != null && Dailiaddresss.ToList().Count > 0)
+ {
+ var Dailiaddress = Dailiaddresss.First();
+ project.Dailiaddress = Dailiaddress.TextContent;
+ }
+ var Dailicontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentLinkTel");
+ if (Dailicontacts != null && Dailicontacts.ToList().Count > 0)
+ {
+ var Dailicontact = Dailicontacts.First();
+ project.Dailicontact = Dailicontact.TextContent;
+ }
+
+
+
+ var zhuanjias = cDocumet.QuerySelectorAll("div").Where(m => m.ClassName == "noticeBidResult-noticeBidResult _notice_content_noticeBidResult-noticeBidResult expert dynamic-form-editor");
+ if (zhuanjias != null && zhuanjias.ToList().Count > 0)
+ {
+ var zhuanjia = zhuanjias.First();
+
+ project.Zhuanjia = zhuanjia.TextContent.Trim();//OuterHtml
}
}
- var Caigourennames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgName");
- if (Caigourennames != null && Caigourennames.ToList().Count > 0)
- {
- var Caigourenname = Caigourennames.First();
- project.Caigourenname = Caigourenname.TextContent;
- }
- var Caigourenaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgAddress");
- if (Caigourenaddresss != null && Caigourenaddresss.ToList().Count > 0)
- {
- var Caigourenaddress = Caigourenaddresss.First();
- project.Caigourenaddress = Caigourenaddress.TextContent;
- }
- var Caigourencontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserLinkTel");
- if (Caigourencontacts != null && Caigourencontacts.ToList().Count > 0)
- {
- var Caigourencontact = Caigourencontacts.First();
- project.Caigourencontact = Caigourencontact.TextContent;
- }
- var Dailinames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agencyName");
- if (Dailinames != null && Dailinames.ToList().Count > 0)
- {
- var Dailiname = Dailinames.First();
- project.Dailiname = Dailiname.TextContent;
- }
- var Dailiaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentAddress");
- if (Dailiaddresss != null && Dailiaddresss.ToList().Count > 0)
- {
- var Dailiaddress = Dailiaddresss.First();
- project.Dailiaddress = Dailiaddress.TextContent;
- }
- var Dailicontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentLinkTel");
- if (Dailicontacts != null && Dailicontacts.ToList().Count > 0)
- {
- var Dailicontact = Dailicontacts.First();
- project.Dailicontact = Dailicontact.TextContent;
- }
+
- var zhuanjias = cDocumet.QuerySelectorAll("div").Where(m => m.ClassName == "noticeBidResult-noticeBidResult _notice_content_noticeBidResult-noticeBidResult expert dynamic-form-editor");
- if (zhuanjias != null && zhuanjias.ToList().Count > 0)
- {
- var zhuanjia = zhuanjias.First();
- project.Zhuanjia = zhuanjia.TextContent.Trim();//OuterHtml
- }
+
project.Creater = "111";
project.Createtime = zhengfuProject.Createtime;
project.Modifier = project.Creater;
@@ -631,32 +935,35 @@
project.RecStatus = "A";
_ccontext1.Projects.Add(project);
-
- var table1 = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_noticeBidResult-noticeBidResult");
-
- if (table1 != null && table1.ToList().Count > 0)
+ if (cDocumet != null)
{
- var table = table1.First().QuerySelectorAll("table");
- if (table != null && table.ToList().Count > 0)
+ //var cDocumet = contentList.First();
+ var table1 = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_noticeBidResult-noticeBidResult");
+
+ if (table1 != null && table1.ToList().Count > 0)
{
- for (int i = 0; i < table.Length; i++)
+ var table = table1.First().QuerySelectorAll("table");
+ if (table != null && table.ToList().Count > 0)
{
- var td = table[i].QuerySelectorAll("td");
- if (td.Length >= 3)
+ for (int i = 0; i < table.Length; i++)
{
-
- var projectzhongbiao = new Projectzhongbiao();
- projectzhongbiao.ZhongbiaoId = Guid.NewGuid().ToString();
- projectzhongbiao.ProjectId = project.Id;
- projectzhongbiao.Names = td[0].TextContent.Trim();
- projectzhongbiao.Price = td[2].TextContent.Trim();
- _ccontext1.Projectzhongbiaos.Add(projectzhongbiao);
+ var td = table[i].QuerySelectorAll("td");
+ if (td.Length >= 3)
+ {
+
+ var projectzhongbiao = new Projectzhongbiao();
+ projectzhongbiao.ZhongbiaoId = Guid.NewGuid().ToString();
+ projectzhongbiao.ProjectId = project.Id;
+ projectzhongbiao.Names = td[0].TextContent.Trim();
+ projectzhongbiao.Price = td[2].TextContent.Trim();
+ _ccontext1.Projectzhongbiaos.Add(projectzhongbiao);
+ }
}
}
+
}
-
}
-
+
@@ -666,13 +973,14 @@
- string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
+
+ string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
string result = string.Empty;
Uri postUrl = new Uri(url);
eswebcrawler eswebcrawler1 = new eswebcrawler();
eswebcrawler1.Id = zhengfuProject.Id;
eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss");
- eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss"):null;
+ eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null;
eswebcrawler1.sheng = zhengfuProject.Sheng;
eswebcrawler1.shengName = ShengName;
eswebcrawler1.city = zhengfuProject.City;
@@ -765,10 +1073,13 @@
foreach (var sichuanjieshoudtl in sichuan.data)
{
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
- ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == pageurl1).FirstOrDefault();
- if (zhengfuProject == null)
+ var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "04" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
+ if (zhengfuProjectCount <= 0)
{
- zhengfuProject = new ZhengfuProject();
+ var zhengfuProject = new ZhengfuProject();
zhengfuProject.Id = Guid.NewGuid().ToString();
zhengfuProject.Gglx = "04";
string gglxName = "鏇存鍏憡";
@@ -1050,10 +1361,14 @@
foreach (var sichuanjieshoudtl in sichuan.data)
{
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
- ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == pageurl1).FirstOrDefault();
- if (zhengfuProject == null)
+ var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
+ var notimestart = DateTime.Parse(notime);
+ var notimeend = notimestart.AddDays(1);
+
+ var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "11" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
+ if (zhengfuProjectCount <= 0)
{
- zhengfuProject = new ZhengfuProject();
+ var zhengfuProject = new ZhengfuProject();
zhengfuProject.Id = Guid.NewGuid().ToString();
zhengfuProject.Gglx = "11";
string gglxName = "搴熸爣鍏憡";
@@ -1235,6 +1550,70 @@
_ccontext.ZhengfuProjects.Add(zhengfuProject);
+ var projects = _ccontext1.Projects.Where(x => x.RecStatus == "A" && x.Number == zhengfuProject.OpenTenderCode && x.Fblx != null ).ToList();
+ if (projects.Count > 0)
+ {
+ foreach (var project in projects)
+ {
+ project.Fblx = "鍋滄璇勫";
+ }
+
+ var parser = new HtmlParser();
+ var document = parser.ParseDocument(zhengfuProject.Content);
+ var contentList = document.All.Where(m => m.Id == "noticeArea");
+ var cDocumet = contentList.First();
+ var Numberss = cDocumet.QuerySelectorAll("p");
+ foreach (var Number in Numberss)
+ {
+
+ if (Number != null && Number.TextContent.Contains("搴熸爣鐞嗙敱"))
+ {
+ if (Number.TextContent.IndexOf("span") >= 0)
+ {
+ var Fbyy = Number.QuerySelector("span");
+ foreach (var project in projects)
+ {
+ project.Fbyy = Fbyy.TextContent.Trim();
+ project.Fblx = "鍋滄璇勫";
+ }
+ }
+
+
+ }
+ if (Number != null && Number.TextContent.Contains("缁堟鍘熷洜"))
+ {
+ if(Number.TextContent.IndexOf("span") >= 0)
+ {
+ var Fbyy = Number.QuerySelector("span");
+ foreach (var project in projects)
+ {
+ project.Fbyy = Fbyy.TextContent.Trim();
+ project.Fblx = "鍋滄璇勫";
+ }
+ }
+
+
+ }
+ }
+ var feibiaoliyous = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_reason-reason");
+ if (feibiaoliyous != null && feibiaoliyous.ToList().Count > 0)
+ {
+ var feibiaoliyou = feibiaoliyous.First();
+ foreach (var project in projects)
+ {
+ project.Fbyy = feibiaoliyou.TextContent.Replace("\n", "").Replace("\t", "");
+ project.Fblx = "缁堟閲囪喘";
+ }
+ }
+
+ _ccontext1.SaveChanges();
+ }
+
+
+
+
+
+
string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
string result = string.Empty;
Uri postUrl = new Uri(url);
@@ -1311,289 +1690,7 @@
}
logg.WriteLog("鍥涘窛鐪佸簾鏍囧叕鍛� 鑾峰彇瀹屾垚", "鍥涘窛");
- //缈婚〉淇℃伅
- currPage = 1;
- pageSize = 10;
- //寮�濮嬭闂� 閲囪喘鍏憡
- logg.WriteLog("鍥涘窛鐪佹剰鍚戝叕寮� 寮�濮嬭幏鍙�", "鍥涘窛");
- while (true)
- {
- logg.WriteLog(currPage.ToString(), "鍥涘窛");
- string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage="+ currPage + "&pageSize="+ pageSize + "¬iceType=59®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime="+ operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
- using (HttpClient client = new HttpClient())
- {
- client.Timeout = TimeSpan.FromSeconds(60);
- HttpResponseMessage response = client.GetAsync(address).Result;
- var res = response.Content.ReadAsStringAsync().Result;
-
- sichuanjieshou sichuan = JsonConvert.DeserializeObject<sichuanjieshou>(res);
-
- //淇濆瓨椤圭洰鍒皊qlserveer鍜宔lasticsearch
- if (sichuan.data != null && sichuan.data.Count > 0)
- {
- foreach (var sichuanjieshoudtl in sichuan.data)
- {
- var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
- ZhengfuProject zhengfuProject = _ccontext.ZhengfuProjects.Where(x => x.Pageurl == pageurl1).FirstOrDefault();
- if (zhengfuProject == null)
- {
- zhengfuProject = new ZhengfuProject();
- zhengfuProject.Id = Guid.NewGuid().ToString();
- zhengfuProject.Gglx = "01";
- string gglxName = "鎰忓悜鍏紑";
- zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime;
- zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime;
- zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode;
- string regionName = sichuanjieshoudtl.regionName;
- zhengfuProject.Sheng = "510000";
- string ShengName = "鍥涘窛鐪�";
- string CityName = "";
- if ("510001" == sichuanjieshoudtl.regionCode)
- {
- zhengfuProject.RegionCode = "";
- regionName = "";
- zhengfuProject.City = "510001";
- CityName = "鐪佹湰绾�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5101"))
- {
- zhengfuProject.City = "510100";
- CityName = "鎴愰兘甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5103"))
- {
- zhengfuProject.City = "510300";
- CityName = "鑷础甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5104"))
- {
- zhengfuProject.City = "510400";
- CityName = "鏀�鏋濊姳甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5105"))
- {
- zhengfuProject.City = "510500";
- CityName = "娉稿窞甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5106"))
- {
- zhengfuProject.City = "510600";
- CityName = "寰烽槼甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5107"))
- {
- zhengfuProject.City = "510700";
- CityName = "缁甸槼甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5108"))
- {
- zhengfuProject.City = "510800";
- CityName = "骞垮厓甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5109"))
- {
- zhengfuProject.City = "510900";
- CityName = "閬傚畞甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5110"))
- {
- zhengfuProject.City = "511000";
- CityName = "鍐呮睙甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5111"))
- {
- zhengfuProject.City = "511100";
- CityName = "涔愬北甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5113"))
- {
- zhengfuProject.City = "511300";
- CityName = "鍗楀厖甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5114"))
- {
- zhengfuProject.City = "5114";
- CityName = "鐪夊北甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5115"))
- {
- zhengfuProject.City = "511500";
- CityName = "瀹滃甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5116"))
- {
- zhengfuProject.City = "511600";
- CityName = "骞垮畨甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5117"))
- {
- zhengfuProject.City = "511700";
- CityName = "杈惧窞甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5118"))
- {
- zhengfuProject.City = "511800";
- CityName = "闆呭畨甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5119"))
- {
- zhengfuProject.City = "511900";
- CityName = "宸翠腑甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5120"))
- {
- zhengfuProject.City = "512000";
- CityName = "璧勯槼甯�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5132"))
- {
- zhengfuProject.City = "513200";
- CityName = "闃垮潩宸�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5133"))
- {
- zhengfuProject.City = "513300";
- CityName = "鐢樺瓬宸�";
- }
- else if (sichuanjieshoudtl.regionCode.StartsWith("5134"))
- {
- zhengfuProject.City = "513400";
- CityName = "鍑夊北宸�";
- }
-
-
- string cgfsName = "";
- switch (sichuanjieshoudtl.purchaseManner)
- {
- case "1":
- zhengfuProject.Cgfs = "1";
- cgfsName = "鍏紑鎷涙爣";
- break;
- case "2":
- zhengfuProject.Cgfs = "2";
- cgfsName = "閭�璇锋嫑鏍�";
- break;
- case "3":
- zhengfuProject.Cgfs = "5";
- cgfsName = "绔炰簤鎬ц皥鍒�";
- break;
- case "4":
- zhengfuProject.Cgfs = "4";
- cgfsName = "璇环";
- break;
- case "5":
- zhengfuProject.Cgfs = "7";
- cgfsName = "鍗曚竴鏉ユ簮";
- break;
- case "6":
- zhengfuProject.Cgfs = "3";
- cgfsName = "绔炰簤鎬х鍟�";
- break;
- case "9":
- zhengfuProject.Cgfs = "10";
- cgfsName = "鍏朵粬";
- break;
- default:
- zhengfuProject.Cgfs = "10";
- cgfsName = "鍏朵粬";
- break;
- }
- zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
- zhengfuProject.Title = sichuanjieshoudtl.title;
- zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle;
- zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl;
- zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser;
-
- if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget))
- {
- zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget);
- }
- zhengfuProject.Agency = sichuanjieshoudtl.agency;
- zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode;
- zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty);
- zhengfuProject.RecStatus = "A";
- zhengfuProject.Creater = "1";
- zhengfuProject.Createtime = DateTime.Now;
- zhengfuProject.Modifier = "1";
- zhengfuProject.Modifytime = zhengfuProject.Createtime;
- _ccontext.ZhengfuProjects.Add(zhengfuProject);
-
-
- string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
- string result = string.Empty;
- Uri postUrl = new Uri(url);
- eswebcrawler eswebcrawler1 = new eswebcrawler();
- eswebcrawler1.Id = zhengfuProject.Id;
- eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss");
- eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null;
- eswebcrawler1.sheng = zhengfuProject.Sheng;
- eswebcrawler1.shengName = ShengName;
- eswebcrawler1.city = zhengfuProject.City;
- eswebcrawler1.cityName = CityName;
- eswebcrawler1.regionCode = zhengfuProject.RegionCode;
- eswebcrawler1.regionName = regionName;
- eswebcrawler1.cgfs = zhengfuProject.Cgfs;
- eswebcrawler1.cgfsName = cgfsName;
- eswebcrawler1.gglx = zhengfuProject.Gglx;
- eswebcrawler1.gglxName = gglxName;
- eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode;
- eswebcrawler1.title = zhengfuProject.Title;
- eswebcrawler1.shorttitle = zhengfuProject.Shorttitle;
- eswebcrawler1.pageurl = zhengfuProject.Pageurl;
- eswebcrawler1.pingmu = zhengfuProject.Pingmu;
- eswebcrawler1.pingmuName = "";
- eswebcrawler1.purchaser = zhengfuProject.Purchaser;
- eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : "";
- eswebcrawler1.agency = zhengfuProject.Agency;
- eswebcrawler1.agencyCode = zhengfuProject.AgencyCode;
- eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss");
-
- string requestJson = JsonConvert.SerializeObject(eswebcrawler1);
-
- using (HttpContent httpContent = new StringContent(requestJson))
- {
- httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
- using (HttpClient httpClient = new HttpClient())
- {
- httpClient.Timeout = TimeSpan.FromSeconds(60);
- HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result;
- result = responseMessage.Content.ReadAsStringAsync().Result;
- }
-
- }
-
- Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result);
-
- if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1")
- {
-
- }
- else
- {
- throw new Exception("淇濆瓨鏁版嵁搴撳嚭閿欙紒");
- }
-
-
- _ccontext.SaveChanges();
- }
- }
- }
-
-
- if (currPage * pageSize >= sichuan.total)
- {
- break;
- }
- else
- {
- currPage += 1;
- }
- Thread.CurrentThread.Join(1000 * 2);//闃绘璁惧畾鏃堕棿
- }
-
-
- }
- logg.WriteLog("鍥涘窛鐪佹剰鍚戝叕寮� 鑾峰彇瀹屾垚", "鍥涘窛");
+
//淇濆瓨鏈�鍚庢洿鏂版椂闂�
if (updatetime != null)
--
Gitblit v1.9.1