using AngleSharp.Html.Parser;
|
using DTO;
|
using Newtonsoft.Json;
|
using System;
|
using System.Collections.Generic;
|
using System.Linq;
|
using System.Net.Http;
|
using System.Net.Http.Headers;
|
using System.Text;
|
using System.Threading;
|
using zhengcaioa.Models;
|
|
namespace Crawler.sichuan
|
{
|
public class sichuantousuoperation
|
{
|
public static string sichuanpageurll = "https://zfcg.scsczt.cn";
|
public static void operations(WebCrawlerContext _ccontext)
|
{
|
try
|
{
|
//获取更新时间
|
string operationStartTime = "";
|
string operationEndTime = "";
|
|
Updatetime updatetime = _ccontext.Updatetimes.Where(x => x.Sheng == "四川省投诉处理").FirstOrDefault();
|
if (updatetime != null)
|
{
|
operationStartTime = updatetime.Updatetime1.ToString("yyyy-MM-dd");
|
operationEndTime = updatetime.Updatetime1.AddMonths(1).ToString("yyyy-MM-dd");
|
}
|
else
|
{
|
operationStartTime = DateTime.Now.AddMonths(-1).ToString("yyyy-MM-dd");
|
operationEndTime = DateTime.Now.ToString("yyyy-MM-dd");
|
}
|
|
//翻页信息
|
int currPage = 1;
|
int pageSize = 10;
|
//开始访问 采购公告
|
logg.WriteLog("四川省投诉处理 开始获取", "四川投诉");
|
var parser = new HtmlParser();
|
while (true)
|
{
|
Thread.CurrentThread.Join(1000 * 5);//阻止设定时间
|
logg.WriteLog(operationStartTime + operationEndTime + currPage.ToString(), "四川投诉");
|
DateTime dateTime = DateTime.Now; // 获取当前UTC时间的DateTime对象
|
long timestamp = (long)(dateTime - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds;
|
string address = "https://www.ccgp-sichuan.gov.cn/gpcms/rest/web/v2/info/selectInfoForIndex?title=®ion=&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=0df9d47e-c210-4176-bce9-ead9845bf311&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=®ionCode=&cityOrArea=&purchaseManner=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "+00:00:00&operationEndTime=" + operationEndTime + "+00:00:00&_t="+ timestamp;//1748999478170
|
sichuantousujieshou sichuan = null;
|
using (HttpClient client = new HttpClient())
|
{
|
client.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage response = client.GetAsync(address).Result;
|
var res = response.Content.ReadAsStringAsync().Result;
|
|
sichuan = JsonConvert.DeserializeObject<sichuantousujieshou>(res);
|
}
|
if(sichuan.data.rows!=null && sichuan.data.rows.Count > 0)
|
{
|
foreach (var sichuanjieshoudtl in sichuan.data.rows)
|
{
|
var Id = Guid.Parse(sichuanjieshoudtl.id);
|
var procurementComplaint = _ccontext.ProcurementComplaints.Find(Id);
|
if (procurementComplaint != null)
|
{
|
continue;
|
}
|
|
|
dateTime = DateTime.Now; // 获取当前UTC时间的DateTime对象
|
timestamp = (long)(dateTime - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds;
|
var askurl = "https://www.ccgp-sichuan.gov.cn/gpcms/rest/web/v2/info/getInfoById?id=" + sichuanjieshoudtl.id +"& _t=" + timestamp;
|
procurementComplaint = new ProcurementComplaint();
|
procurementComplaint.Id = Id;
|
procurementComplaint.ProjectCode = sichuanjieshoudtl.openTenderCode ?? "";
|
if (procurementComplaint.ProjectCode.Length > 50)
|
{
|
procurementComplaint.ProjectCode = procurementComplaint.ProjectCode.Substring(0, 50);
|
}
|
procurementComplaint.ProjectName = "";
|
procurementComplaint.DecisionDate = sichuanjieshoudtl.noticeTime;
|
procurementComplaint.Purchaser = "";
|
procurementComplaint.ProcurementAgency = "";
|
procurementComplaint.Complainant = sichuanjieshoudtl.complainant ?? "";
|
if (procurementComplaint.Complainant.Length > 100)
|
{
|
procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100);
|
}
|
procurementComplaint.ProcurementSupervisionDepartment = sichuanjieshoudtl.author ?? "";
|
if (procurementComplaint.ProcurementSupervisionDepartment.Length > 100)
|
{
|
procurementComplaint.ProcurementSupervisionDepartment = procurementComplaint.ProcurementSupervisionDepartment.Substring(0, 100);
|
}
|
procurementComplaint.CreateTime = DateTime.Now;
|
procurementComplaint.UpdateTime = procurementComplaint.CreateTime;
|
procurementComplaint.Url = "https://www.ccgp-sichuan.gov.cn/maincms-web/article?type=article&id=" + sichuanjieshoudtl.id;
|
|
Thread.CurrentThread.Join(1000 * 5);//阻止设定时间
|
logg.WriteLog(askurl, "四川投诉");
|
sichuantousujieshoumingxi sichuanjieshou = null;
|
using (HttpClient client = new HttpClient())
|
{
|
client.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage response = client.GetAsync(askurl).Result;
|
var res = response.Content.ReadAsStringAsync().Result;
|
|
sichuanjieshou = JsonConvert.DeserializeObject<sichuantousujieshoumingxi>(res);
|
}
|
|
if(sichuanjieshou!=null && sichuanjieshou.code == "200" && !string.IsNullOrEmpty(sichuanjieshou.data.content))
|
{
|
var document = parser.ParseDocument(sichuanjieshou.data.content);
|
// var contentList = document.QuerySelectorAll("span").Where(x=>x.TextContent !="" && !x.InnerHtml.Contains("<span")).ToList();
|
|
var contentList = document.QuerySelectorAll("p").Where(x => x.TextContent != "").ToList();
|
|
var ItemDescriptions = new List<string>();
|
var HandlingResults = new List<string>();
|
var HandlingStatuses = new List<int>();
|
|
foreach (var content in contentList)
|
{
|
var ssss = content.TextContent.Replace(" ","");
|
|
if(ssss.IndexOf("一、项目编号：")>=0|| ssss.IndexOf("一、项目编号:") >= 0)
|
{
|
if (string.IsNullOrEmpty(procurementComplaint.ProjectCode))
|
{
|
procurementComplaint.ProjectCode = ssss.Replace("一、项目编号：","").Replace("一、项目编号:", "").Trim();
|
if (procurementComplaint.ProjectCode.Length > 50)
|
{
|
procurementComplaint.ProjectCode = procurementComplaint.ProjectCode.Substring(0, 50);
|
}
|
}
|
}
|
|
if (ssss.IndexOf("二、项目名称：") >= 0 || ssss.IndexOf("二、项目名称:") >= 0)
|
{
|
if (string.IsNullOrEmpty(procurementComplaint.ProjectName))
|
{
|
procurementComplaint.ProjectName = ssss.Replace("二、项目名称：", "").Replace("二、项目名称:", "").Trim();
|
if (procurementComplaint.ProjectName.Length > 200)
|
{
|
procurementComplaint.ProjectName = procurementComplaint.ProjectName.Substring(0, 200);
|
}
|
}
|
}
|
|
|
if ((ssss.IndexOf("投诉人：") >= 0 || ssss.IndexOf("投诉人:") >= 0) && ssss.IndexOf("被投诉人：") < 0 && ssss.IndexOf("被投诉人：") < 0)
|
{
|
if (string.IsNullOrEmpty(procurementComplaint.Complainant))
|
{
|
procurementComplaint.Complainant = ssss.Replace("投诉人：", "").Replace("投诉人:", "").Trim();
|
if (procurementComplaint.Complainant.Length > 100)
|
{
|
procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100);
|
}
|
}
|
}
|
|
if ((ssss.IndexOf("投诉人1：") >= 0 || ssss.IndexOf("投诉人1:") >= 0) && ssss.IndexOf("被投诉人1：") < 0 && ssss.IndexOf("被投诉人1：") < 0)
|
{
|
if (string.IsNullOrEmpty(procurementComplaint.Complainant))
|
{
|
procurementComplaint.Complainant = ssss.Replace("投诉人1：", "").Replace("投诉人1:", "").Trim();
|
if (procurementComplaint.Complainant.Length > 100)
|
{
|
procurementComplaint.Complainant = procurementComplaint.Complainant.Substring(0, 100);
|
}
|
}
|
}
|
|
if (ssss.IndexOf("被投诉人：") >= 0 || ssss.IndexOf("被投诉人：") >= 0)
|
{
|
if (string.IsNullOrEmpty(procurementComplaint.ProcurementAgency))
|
{
|
procurementComplaint.ProcurementAgency = ssss.Replace("被投诉人：", "").Replace("被投诉人:", "").Trim();
|
if (procurementComplaint.ProcurementAgency.Length > 100)
|
{
|
procurementComplaint.ProcurementAgency = procurementComplaint.ProcurementAgency.Substring(0, 100);
|
}
|
}
|
|
if (string.IsNullOrEmpty(procurementComplaint.Purchaser))
|
{
|
procurementComplaint.Purchaser = ssss.Replace("被投诉人：", "").Replace("被投诉人:", "").Trim();
|
if (procurementComplaint.Purchaser.Length > 100)
|
{
|
procurementComplaint.Purchaser = procurementComplaint.Purchaser.Substring(0, 100);
|
}
|
}
|
}
|
|
|
|
if (ssss.IndexOf("被投诉人1：") >= 0 || ssss.IndexOf("被投诉人1：") >= 0)
|
{
|
if (string.IsNullOrEmpty(procurementComplaint.Purchaser))
|
{
|
procurementComplaint.Purchaser = ssss.Replace("被投诉人1：", "").Replace("被投诉人1:", "").Trim();
|
if (procurementComplaint.Purchaser.Length > 100)
|
{
|
procurementComplaint.Purchaser = procurementComplaint.Purchaser.Substring(0, 100);
|
}
|
}
|
}
|
|
|
|
if (ssss.IndexOf("被投诉人2：") >= 0 || ssss.IndexOf("被投诉人2：") >= 0)
|
{
|
if (string.IsNullOrEmpty(procurementComplaint.ProcurementAgency))
|
{
|
procurementComplaint.ProcurementAgency = ssss.Replace("被投诉人2：", "").Replace("被投诉人2:", "").Trim();
|
if (procurementComplaint.ProcurementAgency.Length > 100)
|
{
|
procurementComplaint.ProcurementAgency = procurementComplaint.ProcurementAgency.Substring(0, 100);
|
}
|
}
|
}
|
|
|
|
if (ssss.IndexOf("投诉事项：") >= 0)
|
{
|
var aaaa = ssss.Replace("投诉事项：", "").Trim();
|
if (ItemDescriptions.Count <= 0)
|
{
|
if (aaaa.Length > 1000)
|
{
|
aaaa = aaaa.Substring(0, 1000);
|
}
|
ItemDescriptions.Add(aaaa);
|
}
|
else
|
{
|
if (aaaa.Length > 1000)
|
{
|
aaaa = aaaa.Substring(0, 1000);
|
}
|
HandlingResults.Add(aaaa);
|
if (aaaa.IndexOf("驳回投诉")>=0)
|
{
|
HandlingStatuses.Add(2);
|
}
|
else if (aaaa.IndexOf("投诉成立") >= 0)
|
{
|
HandlingStatuses.Add(1);
|
|
}
|
else if (aaaa.IndexOf("部分成立") >= 0)
|
{
|
HandlingStatuses.Add(3);
|
}
|
else
|
{
|
HandlingStatuses.Add(0);
|
}
|
|
}
|
}
|
|
|
|
for (int i = 1; i <= 10; i++)
|
{
|
if (ssss.IndexOf("投诉事项" + i + "：") >= 0)
|
{
|
var aaaa = ssss.Replace("投诉事项" + i + "：", "").Trim();
|
if (ItemDescriptions.Count < i)
|
{
|
if (aaaa.Length > 1000)
|
{
|
aaaa = aaaa.Substring(0, 1000);
|
}
|
ItemDescriptions.Add(aaaa);
|
}
|
else
|
{
|
if (aaaa.Length > 1000)
|
{
|
aaaa = aaaa.Substring(0, 1000);
|
}
|
HandlingResults.Add(aaaa);
|
if (aaaa.IndexOf("驳回投诉") >= 0)
|
{
|
HandlingStatuses.Add(2);
|
}
|
else if (aaaa.IndexOf("投诉成立") >= 0)
|
{
|
HandlingStatuses.Add(1);
|
|
}
|
else if (aaaa.IndexOf("部分成立") >= 0)
|
{
|
HandlingStatuses.Add(3);
|
}
|
else
|
{
|
HandlingStatuses.Add(0);
|
}
|
}
|
}
|
}
|
|
|
}
|
|
if (ItemDescriptions.Count == 0)
|
{
|
var bbbb = "";
|
for (int i=0;i< contentList.Count; i++)
|
{
|
var sssssss = contentList[i].TextContent;
|
if (i==0|| sssssss != contentList[i - 1].TextContent)
|
{
|
bbbb += sssssss;
|
}
|
}
|
|
// var bbbb = sichuanjieshou.data.description;
|
if (bbbb.Length > 1000)
|
{
|
bbbb = bbbb.Substring(0, 1000);
|
}
|
ItemDescriptions.Add(bbbb);
|
}
|
if (HandlingResults.Count == 0)
|
{
|
HandlingResults.Add("");
|
HandlingStatuses.Add(0);
|
}
|
|
|
_ccontext.ProcurementComplaints.Add(procurementComplaint);
|
|
|
|
if (ItemDescriptions.Count > 0)
|
{
|
for (int i = 0; i < ItemDescriptions.Count; i++)
|
{
|
var procurementComplaintItems = new ProcurementComplaintItem();
|
procurementComplaintItems.Id = Guid.NewGuid();
|
procurementComplaintItems.ComplaintId = procurementComplaint.Id;
|
procurementComplaintItems.ItemDescription = ItemDescriptions[i];
|
if (HandlingResults.Count == ItemDescriptions.Count)
|
{
|
procurementComplaintItems.HandlingResult = HandlingResults[i];
|
procurementComplaintItems.HandlingStatus = HandlingStatuses[i];
|
}
|
else
|
{
|
procurementComplaintItems.HandlingResult = HandlingResults[0];
|
procurementComplaintItems.HandlingStatus = HandlingStatuses[0];
|
}
|
|
procurementComplaintItems.Sort = (i + 1);
|
_ccontext.ProcurementComplaintItems.Add(procurementComplaintItems);
|
}
|
}
|
_ccontext.SaveChanges();
|
}
|
|
|
|
|
|
}
|
}
|
|
|
if (currPage * pageSize >= sichuan.data.total)
|
{
|
break;
|
}
|
else
|
{
|
currPage += 1;
|
}
|
//Thread.CurrentThread.Join(1000 * 10);//阻止设定时间
|
|
}
|
|
|
logg.WriteLog("四川省投诉处理 获取完成", "四川投诉");
|
//保存最后更新时间
|
if (updatetime != null)
|
{
|
if (updatetime.Updatetime1.AddMonths(1).Date >= DateTime.Now.Date)
|
{
|
updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
|
}
|
else
|
{
|
updatetime.Updatetime1 = updatetime.Updatetime1.AddMonths(1).AddDays(-1);
|
}
|
_ccontext.SaveChanges();
|
}
|
else
|
{
|
updatetime = new Updatetime();
|
updatetime.Id = Guid.NewGuid().ToString();
|
updatetime.Sheng = "四川省";
|
updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
|
_ccontext.Updatetimes.Add(updatetime);
|
_ccontext.SaveChanges();
|
}
|
}
|
catch (Exception ex)
|
{
|
logg.WriteLog(ex.Message, "四川投诉");
|
|
logg.WriteLog(ex.StackTrace, "四川投诉");
|
|
logg.WriteLog(ex.ToString(), "四川投诉");
|
}
|
finally
|
{
|
|
}
|
}
|
}
|
}
|
