New file |
| | |
| | | using AngleSharp.Html.Parser; |
| | | using DTO; |
| | | using Newtonsoft.Json; |
| | | using System; |
| | | using System.Collections.Generic; |
| | | using System.Linq; |
| | | using System.Net.Http; |
| | | using System.Net.Http.Headers; |
| | | using System.Text; |
| | | using System.Threading; |
| | | using zhengcaioa.Models; |
| | | |
| | | namespace Crawler.sichuan |
| | | { |
| | | public class sichuantousuoperation |
| | | { |
| | | public static string sichuanpageurll = "https://zfcg.scsczt.cn"; |
| | | public static void operations(WebCrawlerContext _ccontext) |
| | | { |
| | | try |
| | | { |
| | | //获取更新时间 |
| | | string operationStartTime = ""; |
| | | string operationEndTime = ""; |
| | | |
| | | Updatetime updatetime = _ccontext.Updatetimes.Where(x => x.Sheng == "四川省投诉处理").FirstOrDefault(); |
| | | if (updatetime != null) |
| | | { |
| | | operationStartTime = updatetime.Updatetime1.ToString("yyyy-MM-dd"); |
| | | operationEndTime = updatetime.Updatetime1.AddMonths(1).ToString("yyyy-MM-dd"); |
| | | } |
| | | else |
| | | { |
| | | operationStartTime = DateTime.Now.AddMonths(-1).ToString("yyyy-MM-dd"); |
| | | operationEndTime = DateTime.Now.ToString("yyyy-MM-dd"); |
| | | } |
| | | |
| | | //翻页信息 |
| | | int currPage = 1; |
| | | int pageSize = 10; |
| | | //开始访问 采购公告 |
| | | logg.WriteLog("四川省投诉处理 开始获取", "四川投诉"); |
| | | var parser = new HtmlParser(); |
| | | while (true) |
| | | { |
| | | Thread.CurrentThread.Join(1000 * 5);//阻止设定时间 |
| | | logg.WriteLog(operationStartTime + operationEndTime + currPage.ToString(), "四川投诉"); |
| | | DateTime dateTime = DateTime.Now; // 获取当前UTC时间的DateTime对象 |
| | | long timestamp = (long)(dateTime - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds; |
| | | string address = "https://www.ccgp-sichuan.gov.cn/gpcms/rest/web/v2/info/selectInfoForIndex?title=®ion=&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=0df9d47e-c210-4176-bce9-ead9845bf311&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=®ionCode=&cityOrArea=&purchaseManner=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "+00:00:00&operationEndTime=" + operationEndTime + "+00:00:00&_t="+ timestamp;//1748999478170 |
| | | sichuantousujieshou sichuan = null; |
| | | using (HttpClient client = new HttpClient()) |
| | | { |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage response = client.GetAsync(address).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | |
| | | sichuan = JsonConvert.DeserializeObject<sichuantousujieshou>(res); |
| | | } |
| | | if(sichuan.data.rows!=null && sichuan.data.rows.Count > 0) |
| | | { |
| | | foreach (var sichuanjieshoudtl in sichuan.data.rows) |
| | | { |
| | | var Id = Guid.Parse(sichuanjieshoudtl.id); |
| | | var procurementComplaint = _ccontext.ProcurementComplaints.Find(Id); |
| | | if (procurementComplaint != null) |
| | | { |
| | | continue; |
| | | } |
| | | |
| | | |
| | | dateTime = DateTime.Now; // 获取当前UTC时间的DateTime对象 |
| | | timestamp = (long)(dateTime - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds; |
| | | var askurl = "https://www.ccgp-sichuan.gov.cn/gpcms/rest/web/v2/info/getInfoById?id=" + sichuanjieshoudtl.id +"& _t=" + timestamp; |
| | | procurementComplaint = new ProcurementComplaint(); |
| | | procurementComplaint.Id = Id; |
| | | procurementComplaint.ProjectCode = sichuanjieshoudtl.openTenderCode ?? ""; |
| | | if (procurementComplaint.ProjectCode.Length > 50) |
| | | { |
| | | procurementComplaint.ProjectCode = procurementComplaint.ProjectCode.Substring(0, 50); |
| | | } |
| | | procurementComplaint.ProjectName = ""; |
| | | procurementComplaint.DecisionDate = sichuanjieshoudtl.noticeTime; |
| | | procurementComplaint.Purchaser = ""; |
| | | procurementComplaint.ProcurementAgency = ""; |
| | | procurementComplaint.Complainant = sichuanjieshoudtl.complainant ?? ""; |
| | | if (procurementComplaint.Complainant.Length > 100) |
| | | { |
| | | procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100); |
| | | } |
| | | procurementComplaint.ProcurementSupervisionDepartment = sichuanjieshoudtl.author ?? ""; |
| | | if (procurementComplaint.ProcurementSupervisionDepartment.Length > 100) |
| | | { |
| | | procurementComplaint.ProcurementSupervisionDepartment = procurementComplaint.ProcurementSupervisionDepartment.Substring(0, 100); |
| | | } |
| | | procurementComplaint.CreateTime = DateTime.Now; |
| | | procurementComplaint.UpdateTime = procurementComplaint.CreateTime; |
| | | procurementComplaint.Url = "https://www.ccgp-sichuan.gov.cn/maincms-web/article?type=article&id=" + sichuanjieshoudtl.id; |
| | | |
| | | Thread.CurrentThread.Join(1000 * 5);//阻止设定时间 |
| | | logg.WriteLog(askurl, "四川投诉"); |
| | | sichuantousujieshoumingxi sichuanjieshou = null; |
| | | using (HttpClient client = new HttpClient()) |
| | | { |
| | | client.Timeout = TimeSpan.FromSeconds(60); |
| | | HttpResponseMessage response = client.GetAsync(askurl).Result; |
| | | var res = response.Content.ReadAsStringAsync().Result; |
| | | |
| | | sichuanjieshou = JsonConvert.DeserializeObject<sichuantousujieshoumingxi>(res); |
| | | } |
| | | |
| | | if(sichuanjieshou!=null && sichuanjieshou.code == "200" && !string.IsNullOrEmpty(sichuanjieshou.data.content)) |
| | | { |
| | | var document = parser.ParseDocument(sichuanjieshou.data.content); |
| | | // var contentList = document.QuerySelectorAll("span").Where(x=>x.TextContent !="" && !x.InnerHtml.Contains("<span")).ToList(); |
| | | |
| | | var contentList = document.QuerySelectorAll("p").Where(x => x.TextContent != "").ToList(); |
| | | |
| | | var ItemDescriptions = new List<string>(); |
| | | var HandlingResults = new List<string>(); |
| | | var HandlingStatuses = new List<int>(); |
| | | |
| | | foreach (var content in contentList) |
| | | { |
| | | var ssss = content.TextContent.Replace(" ",""); |
| | | |
| | | if(ssss.IndexOf("一、项目编号:")>=0|| ssss.IndexOf("一、项目编号:") >= 0) |
| | | { |
| | | if (string.IsNullOrEmpty(procurementComplaint.ProjectCode)) |
| | | { |
| | | procurementComplaint.ProjectCode = ssss.Replace("一、项目编号:","").Replace("一、项目编号:", "").Trim(); |
| | | if (procurementComplaint.ProjectCode.Length > 50) |
| | | { |
| | | procurementComplaint.ProjectCode = procurementComplaint.ProjectCode.Substring(0, 50); |
| | | } |
| | | } |
| | | } |
| | | |
| | | if (ssss.IndexOf("二、项目名称:") >= 0 || ssss.IndexOf("二、项目名称:") >= 0) |
| | | { |
| | | if (string.IsNullOrEmpty(procurementComplaint.ProjectName)) |
| | | { |
| | | procurementComplaint.ProjectName = ssss.Replace("二、项目名称:", "").Replace("二、项目名称:", "").Trim(); |
| | | if (procurementComplaint.ProjectName.Length > 200) |
| | | { |
| | | procurementComplaint.ProjectName = procurementComplaint.ProjectName.Substring(0, 200); |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | if ((ssss.IndexOf("投诉人:") >= 0 || ssss.IndexOf("投诉人:") >= 0) && ssss.IndexOf("被投诉人:") < 0 && ssss.IndexOf("被投诉人:") < 0) |
| | | { |
| | | if (string.IsNullOrEmpty(procurementComplaint.Complainant)) |
| | | { |
| | | procurementComplaint.Complainant = ssss.Replace("投诉人:", "").Replace("投诉人:", "").Trim(); |
| | | if (procurementComplaint.Complainant.Length > 100) |
| | | { |
| | | procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100); |
| | | } |
| | | } |
| | | } |
| | | |
| | | if ((ssss.IndexOf("投诉人1:") >= 0 || ssss.IndexOf("投诉人1:") >= 0) && ssss.IndexOf("被投诉人1:") < 0 && ssss.IndexOf("被投诉人1:") < 0) |
| | | { |
| | | if (string.IsNullOrEmpty(procurementComplaint.Complainant)) |
| | | { |
| | | procurementComplaint.Complainant = ssss.Replace("投诉人1:", "").Replace("投诉人1:", "").Trim(); |
| | | if (procurementComplaint.Complainant.Length > 100) |
| | | { |
| | | procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100); |
| | | } |
| | | } |
| | | } |
| | | |
| | | if (ssss.IndexOf("被投诉人:") >= 0 || ssss.IndexOf("被投诉人:") >= 0) |
| | | { |
| | | if (string.IsNullOrEmpty(procurementComplaint.ProcurementAgency)) |
| | | { |
| | | procurementComplaint.ProcurementAgency = ssss.Replace("被投诉人:", "").Replace("被投诉人:", "").Trim(); |
| | | if (procurementComplaint.ProcurementAgency.Length > 100) |
| | | { |
| | | procurementComplaint.ProcurementAgency = procurementComplaint.ProcurementAgency.Substring(0, 100); |
| | | } |
| | | } |
| | | |
| | | if (string.IsNullOrEmpty(procurementComplaint.Purchaser)) |
| | | { |
| | | procurementComplaint.Purchaser = ssss.Replace("被投诉人:", "").Replace("被投诉人:", "").Trim(); |
| | | if (procurementComplaint.Purchaser.Length > 100) |
| | | { |
| | | procurementComplaint.Purchaser = procurementComplaint.Purchaser.Substring(0, 100); |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | |
| | | if (ssss.IndexOf("被投诉人1:") >= 0 || ssss.IndexOf("被投诉人1:") >= 0) |
| | | { |
| | | if (string.IsNullOrEmpty(procurementComplaint.Purchaser)) |
| | | { |
| | | procurementComplaint.Purchaser = ssss.Replace("被投诉人1:", "").Replace("被投诉人1:", "").Trim(); |
| | | if (procurementComplaint.Purchaser.Length > 100) |
| | | { |
| | | procurementComplaint.Purchaser = procurementComplaint.Purchaser.Substring(0, 100); |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | |
| | | if (ssss.IndexOf("被投诉人2:") >= 0 || ssss.IndexOf("被投诉人2:") >= 0) |
| | | { |
| | | if (string.IsNullOrEmpty(procurementComplaint.ProcurementAgency)) |
| | | { |
| | | procurementComplaint.ProcurementAgency = ssss.Replace("被投诉人2:", "").Replace("被投诉人2:", "").Trim(); |
| | | if (procurementComplaint.ProcurementAgency.Length > 100) |
| | | { |
| | | procurementComplaint.ProcurementAgency = procurementComplaint.ProcurementAgency.Substring(0, 100); |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | |
| | | if (ssss.IndexOf("投诉事项:") >= 0) |
| | | { |
| | | var aaaa = ssss.Replace("投诉事项:", "").Trim(); |
| | | if (ItemDescriptions.Count <= 0) |
| | | { |
| | | if (aaaa.Length > 1000) |
| | | { |
| | | aaaa = aaaa.Substring(0, 1000); |
| | | } |
| | | ItemDescriptions.Add(aaaa); |
| | | } |
| | | else |
| | | { |
| | | if (aaaa.Length > 1000) |
| | | { |
| | | aaaa = aaaa.Substring(0, 1000); |
| | | } |
| | | HandlingResults.Add(aaaa); |
| | | if (aaaa.IndexOf("驳回投诉")>=0) |
| | | { |
| | | HandlingStatuses.Add(2); |
| | | } |
| | | else if (aaaa.IndexOf("投诉成立") >= 0) |
| | | { |
| | | HandlingStatuses.Add(1); |
| | | |
| | | } |
| | | else if (aaaa.IndexOf("部分成立") >= 0) |
| | | { |
| | | HandlingStatuses.Add(3); |
| | | } |
| | | else |
| | | { |
| | | HandlingStatuses.Add(0); |
| | | } |
| | | |
| | | } |
| | | } |
| | | |
| | | |
| | | |
| | | for (int i = 1; i <= 10; i++) |
| | | { |
| | | if (ssss.IndexOf("投诉事项" + i + ":") >= 0) |
| | | { |
| | | var aaaa = ssss.Replace("投诉事项" + i + ":", "").Trim(); |
| | | if (ItemDescriptions.Count < i) |
| | | { |
| | | if (aaaa.Length > 1000) |
| | | { |
| | | aaaa = aaaa.Substring(0, 1000); |
| | | } |
| | | ItemDescriptions.Add(aaaa); |
| | | } |
| | | else |
| | | { |
| | | if (aaaa.Length > 1000) |
| | | { |
| | | aaaa = aaaa.Substring(0, 1000); |
| | | } |
| | | HandlingResults.Add(aaaa); |
| | | if (aaaa.IndexOf("驳回投诉") >= 0) |
| | | { |
| | | HandlingStatuses.Add(2); |
| | | } |
| | | else if (aaaa.IndexOf("投诉成立") >= 0) |
| | | { |
| | | HandlingStatuses.Add(1); |
| | | |
| | | } |
| | | else if (aaaa.IndexOf("部分成立") >= 0) |
| | | { |
| | | HandlingStatuses.Add(3); |
| | | } |
| | | else |
| | | { |
| | | HandlingStatuses.Add(0); |
| | | } |
| | | } |
| | | } |
| | | } |
| | | |
| | | |
| | | } |
| | | |
| | | if (ItemDescriptions.Count == 0) |
| | | { |
| | | var bbbb = ""; |
| | | for (int i=0;i< contentList.Count; i++) |
| | | { |
| | | var sssssss = contentList[i].TextContent; |
| | | if (i==0|| sssssss != contentList[i - 1].TextContent) |
| | | { |
| | | bbbb += sssssss; |
| | | } |
| | | } |
| | | |
| | | // var bbbb = sichuanjieshou.data.description; |
| | | if (bbbb.Length > 1000) |
| | | { |
| | | bbbb = bbbb.Substring(0, 1000); |
| | | } |
| | | ItemDescriptions.Add(bbbb); |
| | | } |
| | | if (HandlingResults.Count == 0) |
| | | { |
| | | HandlingResults.Add(""); |
| | | HandlingStatuses.Add(0); |
| | | } |
| | | |
| | | |
| | | _ccontext.ProcurementComplaints.Add(procurementComplaint); |
| | | |
| | | |
| | | |
| | | if (ItemDescriptions.Count > 0) |
| | | { |
| | | for (int i = 0; i < ItemDescriptions.Count; i++) |
| | | { |
| | | var procurementComplaintItems = new ProcurementComplaintItem(); |
| | | procurementComplaintItems.Id = Guid.NewGuid(); |
| | | procurementComplaintItems.ComplaintId = procurementComplaint.Id; |
| | | procurementComplaintItems.ItemDescription = ItemDescriptions[i]; |
| | | if (HandlingResults.Count == ItemDescriptions.Count) |
| | | { |
| | | procurementComplaintItems.HandlingResult = HandlingResults[i]; |
| | | procurementComplaintItems.HandlingStatus = HandlingStatuses[i]; |
| | | } |
| | | else |
| | | { |
| | | procurementComplaintItems.HandlingResult = HandlingResults[0]; |
| | | procurementComplaintItems.HandlingStatus = HandlingStatuses[0]; |
| | | } |
| | | |
| | | procurementComplaintItems.Sort = (i + 1); |
| | | _ccontext.ProcurementComplaintItems.Add(procurementComplaintItems); |
| | | } |
| | | } |
| | | _ccontext.SaveChanges(); |
| | | } |
| | | |
| | | |
| | | |
| | | |
| | | |
| | | } |
| | | } |
| | | |
| | | |
| | | if (currPage * pageSize >= sichuan.data.total) |
| | | { |
| | | break; |
| | | } |
| | | else |
| | | { |
| | | currPage += 1; |
| | | } |
| | | //Thread.CurrentThread.Join(1000 * 10);//阻止设定时间 |
| | | |
| | | } |
| | | |
| | | |
| | | logg.WriteLog("四川省投诉处理 获取完成", "四川投诉"); |
| | | //保存最后更新时间 |
| | | if (updatetime != null) |
| | | { |
| | | if (updatetime.Updatetime1.AddMonths(1).Date >= DateTime.Now.Date) |
| | | { |
| | | updatetime.Updatetime1 = DateTime.Now.AddDays(-1); |
| | | } |
| | | else |
| | | { |
| | | updatetime.Updatetime1 = updatetime.Updatetime1.AddMonths(1).AddDays(-1); |
| | | } |
| | | _ccontext.SaveChanges(); |
| | | } |
| | | else |
| | | { |
| | | updatetime = new Updatetime(); |
| | | updatetime.Id = Guid.NewGuid().ToString(); |
| | | updatetime.Sheng = "四川省"; |
| | | updatetime.Updatetime1 = DateTime.Now.AddDays(-1); |
| | | _ccontext.Updatetimes.Add(updatetime); |
| | | _ccontext.SaveChanges(); |
| | | } |
| | | } |
| | | catch (Exception ex) |
| | | { |
| | | logg.WriteLog(ex.Message, "四川投诉"); |
| | | |
| | | logg.WriteLog(ex.StackTrace, "四川投诉"); |
| | | |
| | | logg.WriteLog(ex.ToString(), "四川投诉"); |
| | | } |
| | | finally |
| | | { |
| | | |
| | | } |
| | | } |
| | | } |
| | | } |