using AngleSharp.Html.Parser; using DTO; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; using zhengcaioa.Models; namespace Crawler.sichuan { public class sichuantousuoperation { public static string sichuanpageurll = "https://zfcg.scsczt.cn"; public static void operations(WebCrawlerContext _ccontext) { try { //获取更新时间 string operationStartTime = ""; string operationEndTime = ""; Updatetime updatetime = _ccontext.Updatetimes.Where(x => x.Sheng == "四川省投诉处理").FirstOrDefault(); if (updatetime != null) { operationStartTime = updatetime.Updatetime1.ToString("yyyy-MM-dd"); operationEndTime = updatetime.Updatetime1.AddMonths(1).ToString("yyyy-MM-dd"); } else { operationStartTime = DateTime.Now.AddMonths(-1).ToString("yyyy-MM-dd"); operationEndTime = DateTime.Now.ToString("yyyy-MM-dd"); } //翻页信息 int currPage = 1; int pageSize = 10; //开始访问 采购公告 logg.WriteLog("四川省投诉处理 开始获取", "四川投诉"); var parser = new HtmlParser(); while (true) { Thread.CurrentThread.Join(1000 * 5);//阻止设定时间 logg.WriteLog(operationStartTime + operationEndTime + currPage.ToString(), "四川投诉"); DateTime dateTime = DateTime.Now; // 获取当前UTC时间的DateTime对象 long timestamp = (long)(dateTime - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds; string address = "https://www.ccgp-sichuan.gov.cn/gpcms/rest/web/v2/info/selectInfoForIndex?title=®ion=&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=0df9d47e-c210-4176-bce9-ead9845bf311&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=®ionCode=&cityOrArea=&purchaseManner=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "+00:00:00&operationEndTime=" + operationEndTime + "+00:00:00&_t="+ timestamp;//1748999478170 sichuantousujieshou sichuan = null; using (HttpClient client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage response = client.GetAsync(address).Result; var res = response.Content.ReadAsStringAsync().Result; sichuan = JsonConvert.DeserializeObject(res); } if(sichuan.data.rows!=null && sichuan.data.rows.Count > 0) { foreach (var sichuanjieshoudtl in sichuan.data.rows) { var Id = Guid.Parse(sichuanjieshoudtl.id); var procurementComplaint = _ccontext.ProcurementComplaints.Find(Id); if (procurementComplaint != null) { continue; } dateTime = DateTime.Now; // 获取当前UTC时间的DateTime对象 timestamp = (long)(dateTime - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds; var askurl = "https://www.ccgp-sichuan.gov.cn/gpcms/rest/web/v2/info/getInfoById?id=" + sichuanjieshoudtl.id +"& _t=" + timestamp; procurementComplaint = new ProcurementComplaint(); procurementComplaint.Id = Id; procurementComplaint.ProjectCode = sichuanjieshoudtl.openTenderCode ?? ""; if (procurementComplaint.ProjectCode.Length > 50) { procurementComplaint.ProjectCode = procurementComplaint.ProjectCode.Substring(0, 50); } procurementComplaint.ProjectName = ""; procurementComplaint.DecisionDate = sichuanjieshoudtl.noticeTime; procurementComplaint.Purchaser = ""; procurementComplaint.ProcurementAgency = ""; procurementComplaint.Complainant = sichuanjieshoudtl.complainant ?? ""; if (procurementComplaint.Complainant.Length > 100) { procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100); } procurementComplaint.ProcurementSupervisionDepartment = sichuanjieshoudtl.author ?? ""; if (procurementComplaint.ProcurementSupervisionDepartment.Length > 100) { procurementComplaint.ProcurementSupervisionDepartment = procurementComplaint.ProcurementSupervisionDepartment.Substring(0, 100); } procurementComplaint.CreateTime = DateTime.Now; procurementComplaint.UpdateTime = procurementComplaint.CreateTime; procurementComplaint.Url = "https://www.ccgp-sichuan.gov.cn/maincms-web/article?type=article&id=" + sichuanjieshoudtl.id; Thread.CurrentThread.Join(1000 * 5);//阻止设定时间 logg.WriteLog(askurl, "四川投诉"); sichuantousujieshoumingxi sichuanjieshou = null; using (HttpClient client = new HttpClient()) { client.Timeout = TimeSpan.FromSeconds(60); HttpResponseMessage response = client.GetAsync(askurl).Result; var res = response.Content.ReadAsStringAsync().Result; sichuanjieshou = JsonConvert.DeserializeObject(res); } if(sichuanjieshou!=null && sichuanjieshou.code == "200" && !string.IsNullOrEmpty(sichuanjieshou.data.content)) { var document = parser.ParseDocument(sichuanjieshou.data.content); // var contentList = document.QuerySelectorAll("span").Where(x=>x.TextContent !="" && !x.InnerHtml.Contains(" x.TextContent != "").ToList(); var ItemDescriptions = new List(); var HandlingResults = new List(); var HandlingStatuses = new List(); foreach (var content in contentList) { var ssss = content.TextContent.Replace(" ",""); if(ssss.IndexOf("一、项目编号:")>=0|| ssss.IndexOf("一、项目编号:") >= 0) { if (string.IsNullOrEmpty(procurementComplaint.ProjectCode)) { procurementComplaint.ProjectCode = ssss.Replace("一、项目编号:","").Replace("一、项目编号:", "").Trim(); if (procurementComplaint.ProjectCode.Length > 50) { procurementComplaint.ProjectCode = procurementComplaint.ProjectCode.Substring(0, 50); } } } if (ssss.IndexOf("二、项目名称:") >= 0 || ssss.IndexOf("二、项目名称:") >= 0) { if (string.IsNullOrEmpty(procurementComplaint.ProjectName)) { procurementComplaint.ProjectName = ssss.Replace("二、项目名称:", "").Replace("二、项目名称:", "").Trim(); if (procurementComplaint.ProjectName.Length > 200) { procurementComplaint.ProjectName = procurementComplaint.ProjectName.Substring(0, 200); } } } if ((ssss.IndexOf("投诉人:") >= 0 || ssss.IndexOf("投诉人:") >= 0) && ssss.IndexOf("被投诉人:") < 0 && ssss.IndexOf("被投诉人:") < 0) { if (string.IsNullOrEmpty(procurementComplaint.Complainant)) { procurementComplaint.Complainant = ssss.Replace("投诉人:", "").Replace("投诉人:", "").Trim(); if (procurementComplaint.Complainant.Length > 100) { procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100); } } } if ((ssss.IndexOf("投诉人1:") >= 0 || ssss.IndexOf("投诉人1:") >= 0) && ssss.IndexOf("被投诉人1:") < 0 && ssss.IndexOf("被投诉人1:") < 0) { if (string.IsNullOrEmpty(procurementComplaint.Complainant)) { procurementComplaint.Complainant = ssss.Replace("投诉人1:", "").Replace("投诉人1:", "").Trim(); if (procurementComplaint.Complainant.Length > 100) { procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100); } } } if (ssss.IndexOf("被投诉人:") >= 0 || ssss.IndexOf("被投诉人:") >= 0) { if (string.IsNullOrEmpty(procurementComplaint.ProcurementAgency)) { procurementComplaint.ProcurementAgency = ssss.Replace("被投诉人:", "").Replace("被投诉人:", "").Trim(); if (procurementComplaint.ProcurementAgency.Length > 100) { procurementComplaint.ProcurementAgency = procurementComplaint.ProcurementAgency.Substring(0, 100); } } if (string.IsNullOrEmpty(procurementComplaint.Purchaser)) { procurementComplaint.Purchaser = ssss.Replace("被投诉人:", "").Replace("被投诉人:", "").Trim(); if (procurementComplaint.Purchaser.Length > 100) { procurementComplaint.Purchaser = procurementComplaint.Purchaser.Substring(0, 100); } } } if (ssss.IndexOf("被投诉人1:") >= 0 || ssss.IndexOf("被投诉人1:") >= 0) { if (string.IsNullOrEmpty(procurementComplaint.Purchaser)) { procurementComplaint.Purchaser = ssss.Replace("被投诉人1:", "").Replace("被投诉人1:", "").Trim(); if (procurementComplaint.Purchaser.Length > 100) { procurementComplaint.Purchaser = procurementComplaint.Purchaser.Substring(0, 100); } } } if (ssss.IndexOf("被投诉人2:") >= 0 || ssss.IndexOf("被投诉人2:") >= 0) { if (string.IsNullOrEmpty(procurementComplaint.ProcurementAgency)) { procurementComplaint.ProcurementAgency = ssss.Replace("被投诉人2:", "").Replace("被投诉人2:", "").Trim(); if (procurementComplaint.ProcurementAgency.Length > 100) { procurementComplaint.ProcurementAgency = procurementComplaint.ProcurementAgency.Substring(0, 100); } } } if (ssss.IndexOf("投诉事项:") >= 0) { var aaaa = ssss.Replace("投诉事项:", "").Trim(); if (ItemDescriptions.Count <= 0) { if (aaaa.Length > 1000) { aaaa = aaaa.Substring(0, 1000); } ItemDescriptions.Add(aaaa); } else { if (aaaa.Length > 1000) { aaaa = aaaa.Substring(0, 1000); } HandlingResults.Add(aaaa); if (aaaa.IndexOf("驳回投诉")>=0) { HandlingStatuses.Add(2); } else if (aaaa.IndexOf("投诉成立") >= 0) { HandlingStatuses.Add(1); } else if (aaaa.IndexOf("部分成立") >= 0) { HandlingStatuses.Add(3); } else { HandlingStatuses.Add(0); } } } for (int i = 1; i <= 10; i++) { if (ssss.IndexOf("投诉事项" + i + ":") >= 0) { var aaaa = ssss.Replace("投诉事项" + i + ":", "").Trim(); if (ItemDescriptions.Count < i) { if (aaaa.Length > 1000) { aaaa = aaaa.Substring(0, 1000); } ItemDescriptions.Add(aaaa); } else { if (aaaa.Length > 1000) { aaaa = aaaa.Substring(0, 1000); } HandlingResults.Add(aaaa); if (aaaa.IndexOf("驳回投诉") >= 0) { HandlingStatuses.Add(2); } else if (aaaa.IndexOf("投诉成立") >= 0) { HandlingStatuses.Add(1); } else if (aaaa.IndexOf("部分成立") >= 0) { HandlingStatuses.Add(3); } else { HandlingStatuses.Add(0); } } } } } if (ItemDescriptions.Count == 0) { var bbbb = ""; for (int i=0;i< contentList.Count; i++) { var sssssss = contentList[i].TextContent; if (i==0|| sssssss != contentList[i - 1].TextContent) { bbbb += sssssss; } } // var bbbb = sichuanjieshou.data.description; if (bbbb.Length > 1000) { bbbb = bbbb.Substring(0, 1000); } ItemDescriptions.Add(bbbb); } if (HandlingResults.Count == 0) { HandlingResults.Add(""); HandlingStatuses.Add(0); } _ccontext.ProcurementComplaints.Add(procurementComplaint); if (ItemDescriptions.Count > 0) { for (int i = 0; i < ItemDescriptions.Count; i++) { var procurementComplaintItems = new ProcurementComplaintItem(); procurementComplaintItems.Id = Guid.NewGuid(); procurementComplaintItems.ComplaintId = procurementComplaint.Id; procurementComplaintItems.ItemDescription = ItemDescriptions[i]; if (HandlingResults.Count == ItemDescriptions.Count) { procurementComplaintItems.HandlingResult = HandlingResults[i]; procurementComplaintItems.HandlingStatus = HandlingStatuses[i]; } else { procurementComplaintItems.HandlingResult = HandlingResults[0]; procurementComplaintItems.HandlingStatus = HandlingStatuses[0]; } procurementComplaintItems.Sort = (i + 1); _ccontext.ProcurementComplaintItems.Add(procurementComplaintItems); } } _ccontext.SaveChanges(); } } } if (currPage * pageSize >= sichuan.data.total) { break; } else { currPage += 1; } //Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 } logg.WriteLog("四川省投诉处理 获取完成", "四川投诉"); //保存最后更新时间 if (updatetime != null) { if (updatetime.Updatetime1.AddMonths(1).Date >= DateTime.Now.Date) { updatetime.Updatetime1 = DateTime.Now.AddDays(-1); } else { updatetime.Updatetime1 = updatetime.Updatetime1.AddMonths(1).AddDays(-1); } _ccontext.SaveChanges(); } else { updatetime = new Updatetime(); updatetime.Id = Guid.NewGuid().ToString(); updatetime.Sheng = "四川省"; updatetime.Updatetime1 = DateTime.Now.AddDays(-1); _ccontext.Updatetimes.Add(updatetime); _ccontext.SaveChanges(); } } catch (Exception ex) { logg.WriteLog(ex.Message, "四川投诉"); logg.WriteLog(ex.StackTrace, "四川投诉"); logg.WriteLog(ex.ToString(), "四川投诉"); } finally { } } } }