using AngleSharp.Html.Parser;
|
using DTO;
|
using Newtonsoft.Json;
|
using System;
|
using System.Collections.Generic;
|
using System.Linq;
|
using System.Net.Http;
|
using System.Net.Http.Headers;
|
using System.Text;
|
using System.Threading;
|
using zhengcaioa.Models;
|
|
namespace Crawler.sichuan
|
{
|
public class sichuanoperation
|
{
|
|
public static string sichuanpageurll = "https://zfcg.scsczt.cn";
|
public static void operations(WebCrawlerContext _ccontext, zhengcaioaContext _ccontext1)
|
{
|
try
|
{
|
|
//获取更新时间
|
string operationStartTime = "";
|
Updatetime updatetime = _ccontext.Updatetimes.Where(x=>x.Sheng=="四川省").FirstOrDefault();
|
if (updatetime != null)
|
{
|
operationStartTime = updatetime.Updatetime1.ToString("yyyy-MM-dd")+ "%2000:00:00";
|
//updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
|
//_ccontext.SaveChanges();
|
}
|
//else
|
//{
|
// updatetime = new Updatetime();
|
// updatetime.Id = Guid.NewGuid().ToString();
|
// updatetime.Sheng = "四川省";
|
// updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
|
// _ccontext.Updatetimes.Add(updatetime);
|
// _ccontext.SaveChanges();
|
//}
|
|
//翻页信息
|
int currPage = 1;
|
int pageSize = 10;
|
//开始访问 采购公告
|
logg.WriteLog("四川省意向公开 开始获取", "四川");
|
while (true)
|
{
|
logg.WriteLog(currPage.ToString(), "四川");
|
string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=59®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
|
using (HttpClient client = new HttpClient())
|
{
|
client.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage response = client.GetAsync(address).Result;
|
var res = response.Content.ReadAsStringAsync().Result;
|
|
sichuanjieshou sichuan = JsonConvert.DeserializeObject<sichuanjieshou>(res);
|
|
//保存项目到sqlserveer和elasticsearch
|
if (sichuan.data != null && sichuan.data.Count > 0)
|
{
|
foreach (var sichuanjieshoudtl in sichuan.data)
|
{
|
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
|
var notimestart = DateTime.Parse(notime);
|
var notimeend = notimestart.AddDays(1);
|
|
var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "01" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
|
if (zhengfuProjectCount <= 0)
|
{
|
var zhengfuProject = new ZhengfuProject();
|
zhengfuProject.Id = Guid.NewGuid().ToString();
|
zhengfuProject.Gglx = "01";
|
string gglxName = "意向公开";
|
zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime;
|
zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime;
|
zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode;
|
string regionName = sichuanjieshoudtl.regionName;
|
zhengfuProject.Sheng = "510000";
|
string ShengName = "四川省";
|
string CityName = "";
|
if ("510001" == sichuanjieshoudtl.regionCode)
|
{
|
zhengfuProject.RegionCode = "";
|
regionName = "";
|
zhengfuProject.City = "510001";
|
CityName = "省本级";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5101"))
|
{
|
zhengfuProject.City = "510100";
|
CityName = "成都市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5103"))
|
{
|
zhengfuProject.City = "510300";
|
CityName = "自贡市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5104"))
|
{
|
zhengfuProject.City = "510400";
|
CityName = "攀枝花市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5105"))
|
{
|
zhengfuProject.City = "510500";
|
CityName = "泸州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5106"))
|
{
|
zhengfuProject.City = "510600";
|
CityName = "德阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5107"))
|
{
|
zhengfuProject.City = "510700";
|
CityName = "绵阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5108"))
|
{
|
zhengfuProject.City = "510800";
|
CityName = "广元市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5109"))
|
{
|
zhengfuProject.City = "510900";
|
CityName = "遂宁市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5110"))
|
{
|
zhengfuProject.City = "511000";
|
CityName = "内江市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5111"))
|
{
|
zhengfuProject.City = "511100";
|
CityName = "乐山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5113"))
|
{
|
zhengfuProject.City = "511300";
|
CityName = "南充市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5114"))
|
{
|
zhengfuProject.City = "5114";
|
CityName = "眉山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5115"))
|
{
|
zhengfuProject.City = "511500";
|
CityName = "宜宾市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5116"))
|
{
|
zhengfuProject.City = "511600";
|
CityName = "广安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5117"))
|
{
|
zhengfuProject.City = "511700";
|
CityName = "达州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5118"))
|
{
|
zhengfuProject.City = "511800";
|
CityName = "雅安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5119"))
|
{
|
zhengfuProject.City = "511900";
|
CityName = "巴中市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5120"))
|
{
|
zhengfuProject.City = "512000";
|
CityName = "资阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5132"))
|
{
|
zhengfuProject.City = "513200";
|
CityName = "阿坝州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5133"))
|
{
|
zhengfuProject.City = "513300";
|
CityName = "甘孜州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5134"))
|
{
|
zhengfuProject.City = "513400";
|
CityName = "凉山州";
|
}
|
|
|
string cgfsName = "";
|
switch (sichuanjieshoudtl.purchaseManner)
|
{
|
case "1":
|
zhengfuProject.Cgfs = "1";
|
cgfsName = "公开招标";
|
break;
|
case "2":
|
zhengfuProject.Cgfs = "2";
|
cgfsName = "邀请招标";
|
break;
|
case "3":
|
zhengfuProject.Cgfs = "5";
|
cgfsName = "竞争性谈判";
|
break;
|
case "4":
|
zhengfuProject.Cgfs = "4";
|
cgfsName = "询价";
|
break;
|
case "5":
|
zhengfuProject.Cgfs = "7";
|
cgfsName = "单一来源";
|
break;
|
case "6":
|
zhengfuProject.Cgfs = "3";
|
cgfsName = "竞争性磋商";
|
break;
|
case "9":
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
break;
|
default:
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
break;
|
}
|
zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
|
zhengfuProject.Title = sichuanjieshoudtl.title;
|
zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle;
|
zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser;
|
|
if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget))
|
{
|
zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget);
|
}
|
zhengfuProject.Agency = sichuanjieshoudtl.agency;
|
zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode;
|
zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty);
|
zhengfuProject.RecStatus = "A";
|
zhengfuProject.Creater = "1";
|
zhengfuProject.Createtime = DateTime.Now;
|
zhengfuProject.Modifier = "1";
|
zhengfuProject.Modifytime = zhengfuProject.Createtime;
|
_ccontext.ZhengfuProjects.Add(zhengfuProject);
|
|
|
string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
|
string result = string.Empty;
|
Uri postUrl = new Uri(url);
|
eswebcrawler eswebcrawler1 = new eswebcrawler();
|
eswebcrawler1.Id = zhengfuProject.Id;
|
eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss");
|
eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null;
|
eswebcrawler1.sheng = zhengfuProject.Sheng;
|
eswebcrawler1.shengName = ShengName;
|
eswebcrawler1.city = zhengfuProject.City;
|
eswebcrawler1.cityName = CityName;
|
eswebcrawler1.regionCode = zhengfuProject.RegionCode;
|
eswebcrawler1.regionName = regionName;
|
eswebcrawler1.cgfs = zhengfuProject.Cgfs;
|
eswebcrawler1.cgfsName = cgfsName;
|
eswebcrawler1.gglx = zhengfuProject.Gglx;
|
eswebcrawler1.gglxName = gglxName;
|
eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode;
|
eswebcrawler1.title = zhengfuProject.Title;
|
eswebcrawler1.shorttitle = zhengfuProject.Shorttitle;
|
eswebcrawler1.pageurl = zhengfuProject.Pageurl;
|
eswebcrawler1.pingmu = zhengfuProject.Pingmu;
|
eswebcrawler1.pingmuName = "";
|
eswebcrawler1.purchaser = zhengfuProject.Purchaser;
|
eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : "";
|
eswebcrawler1.agency = zhengfuProject.Agency;
|
eswebcrawler1.agencyCode = zhengfuProject.AgencyCode;
|
eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss");
|
|
string requestJson = JsonConvert.SerializeObject(eswebcrawler1);
|
|
using (HttpContent httpContent = new StringContent(requestJson))
|
{
|
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
|
using (HttpClient httpClient = new HttpClient())
|
{
|
httpClient.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result;
|
result = responseMessage.Content.ReadAsStringAsync().Result;
|
}
|
|
}
|
|
Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result);
|
|
if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1")
|
{
|
|
}
|
else
|
{
|
throw new Exception("保存数据库出错！");
|
}
|
|
|
_ccontext.SaveChanges();
|
}
|
}
|
}
|
|
|
if (currPage * pageSize >= sichuan.total)
|
{
|
break;
|
}
|
else
|
{
|
currPage += 1;
|
}
|
Thread.CurrentThread.Join(1000 * 20);//阻止设定时间
|
}
|
|
|
}
|
logg.WriteLog("四川省意向公开 获取完成", "四川");
|
|
//翻页信息
|
currPage = 1;
|
pageSize = 10;
|
//开始访问 采购公告
|
logg.WriteLog("四川省采购公告 开始获取", "四川");
|
while (true)
|
{
|
logg.WriteLog(currPage.ToString(), "四川");
|
string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=00101®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
|
using (HttpClient client = new HttpClient())
|
{
|
client.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage response = client.GetAsync(address).Result;
|
var res = response.Content.ReadAsStringAsync().Result;
|
|
sichuanjieshou sichuan = JsonConvert.DeserializeObject<sichuanjieshou>(res);
|
|
//保存项目到sqlserveer和elasticsearch
|
if (sichuan.data != null && sichuan.data.Count > 0)
|
{
|
foreach (var sichuanjieshoudtl in sichuan.data)
|
{
|
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
|
var notimestart = DateTime.Parse(notime);
|
var notimeend = notimestart.AddDays(1);
|
var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "02" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
|
if (zhengfuProjectCount <= 0)
|
{
|
var zhengfuProject = new ZhengfuProject();
|
zhengfuProject.Id = Guid.NewGuid().ToString();
|
zhengfuProject.Gglx = "02";
|
string gglxName = "采购公告";
|
zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime;
|
zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime;
|
zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode;
|
string regionName = sichuanjieshoudtl.regionName;
|
zhengfuProject.Sheng = "510000";
|
string ShengName = "四川省";
|
string CityName = "";
|
if ("510001" == sichuanjieshoudtl.regionCode)
|
{
|
zhengfuProject.RegionCode = "";
|
regionName = "";
|
zhengfuProject.City = "510001";
|
CityName = "省本级";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5101"))
|
{
|
zhengfuProject.City = "510100";
|
CityName = "成都市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5103"))
|
{
|
zhengfuProject.City = "510300";
|
CityName = "自贡市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5104"))
|
{
|
zhengfuProject.City = "510400";
|
CityName = "攀枝花市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5105"))
|
{
|
zhengfuProject.City = "510500";
|
CityName = "泸州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5106"))
|
{
|
zhengfuProject.City = "510600";
|
CityName = "德阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5107"))
|
{
|
zhengfuProject.City = "510700";
|
CityName = "绵阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5108"))
|
{
|
zhengfuProject.City = "510800";
|
CityName = "广元市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5109"))
|
{
|
zhengfuProject.City = "510900";
|
CityName = "遂宁市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5110"))
|
{
|
zhengfuProject.City = "511000";
|
CityName = "内江市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5111"))
|
{
|
zhengfuProject.City = "511100";
|
CityName = "乐山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5113"))
|
{
|
zhengfuProject.City = "511300";
|
CityName = "南充市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5114"))
|
{
|
zhengfuProject.City = "5114";
|
CityName = "眉山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5115"))
|
{
|
zhengfuProject.City = "511500";
|
CityName = "宜宾市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5116"))
|
{
|
zhengfuProject.City = "511600";
|
CityName = "广安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5117"))
|
{
|
zhengfuProject.City = "511700";
|
CityName = "达州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5118"))
|
{
|
zhengfuProject.City = "511800";
|
CityName = "雅安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5119"))
|
{
|
zhengfuProject.City = "511900";
|
CityName = "巴中市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5120"))
|
{
|
zhengfuProject.City = "512000";
|
CityName = "资阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5132"))
|
{
|
zhengfuProject.City = "513200";
|
CityName = "阿坝州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5133"))
|
{
|
zhengfuProject.City = "513300";
|
CityName = "甘孜州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5134"))
|
{
|
zhengfuProject.City = "513400";
|
CityName = "凉山州";
|
}
|
|
|
string cgfsName = "";
|
switch (sichuanjieshoudtl.purchaseManner)
|
{
|
case "1":
|
zhengfuProject.Cgfs = "1";
|
cgfsName = "公开招标";
|
break;
|
case "2":
|
zhengfuProject.Cgfs = "2";
|
cgfsName = "邀请招标";
|
break;
|
case "3":
|
zhengfuProject.Cgfs = "5";
|
cgfsName = "竞争性谈判";
|
break;
|
case "4":
|
zhengfuProject.Cgfs = "4";
|
cgfsName = "询价";
|
break;
|
case "5":
|
zhengfuProject.Cgfs = "7";
|
cgfsName = "单一来源";
|
break;
|
case "6":
|
zhengfuProject.Cgfs = "3";
|
cgfsName = "竞争性磋商";
|
break;
|
case "9":
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
break;
|
default:
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
break;
|
}
|
zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
|
zhengfuProject.Title = sichuanjieshoudtl.title;
|
zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle;
|
zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser;
|
|
if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget))
|
{
|
zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget);
|
}
|
zhengfuProject.Agency = sichuanjieshoudtl.agency;
|
zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode;
|
zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty);
|
zhengfuProject.RecStatus = "A";
|
zhengfuProject.Creater = "1";
|
zhengfuProject.Createtime = DateTime.Now;
|
zhengfuProject.Modifier = "1";
|
zhengfuProject.Modifytime = zhengfuProject.Createtime;
|
_ccontext.ZhengfuProjects.Add(zhengfuProject);
|
|
|
string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
|
string result = string.Empty;
|
Uri postUrl = new Uri(url);
|
eswebcrawler eswebcrawler1 = new eswebcrawler();
|
eswebcrawler1.Id = zhengfuProject.Id;
|
eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss");
|
eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null;
|
eswebcrawler1.sheng = zhengfuProject.Sheng;
|
eswebcrawler1.shengName = ShengName;
|
eswebcrawler1.city = zhengfuProject.City;
|
eswebcrawler1.cityName = CityName;
|
eswebcrawler1.regionCode = zhengfuProject.RegionCode;
|
eswebcrawler1.regionName = regionName;
|
eswebcrawler1.cgfs = zhengfuProject.Cgfs;
|
eswebcrawler1.cgfsName = cgfsName;
|
eswebcrawler1.gglx = zhengfuProject.Gglx;
|
eswebcrawler1.gglxName = gglxName;
|
eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode;
|
eswebcrawler1.title = zhengfuProject.Title;
|
eswebcrawler1.shorttitle = zhengfuProject.Shorttitle;
|
eswebcrawler1.pageurl = zhengfuProject.Pageurl;
|
eswebcrawler1.pingmu = zhengfuProject.Pingmu;
|
eswebcrawler1.pingmuName = "";
|
eswebcrawler1.purchaser = zhengfuProject.Purchaser;
|
eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : "";
|
eswebcrawler1.agency = zhengfuProject.Agency;
|
eswebcrawler1.agencyCode = zhengfuProject.AgencyCode;
|
eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss");
|
|
string requestJson = JsonConvert.SerializeObject(eswebcrawler1);
|
|
using (HttpContent httpContent = new StringContent(requestJson))
|
{
|
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
|
using (HttpClient httpClient = new HttpClient())
|
{
|
httpClient.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result;
|
result = responseMessage.Content.ReadAsStringAsync().Result;
|
}
|
|
}
|
|
Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result);
|
|
if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1")
|
{
|
|
}
|
else
|
{
|
throw new Exception("保存数据库出错！");
|
}
|
|
|
_ccontext.SaveChanges();
|
}
|
}
|
}
|
|
|
if (currPage * pageSize >= sichuan.total)
|
{
|
break;
|
}
|
else
|
{
|
currPage += 1;
|
}
|
Thread.CurrentThread.Join(1000 * 20);//阻止设定时间
|
}
|
|
|
}
|
logg.WriteLog("四川省采购公告 获取完成", "四川");
|
|
//翻页信息
|
currPage = 1;
|
pageSize = 10;
|
//开始访问 采购公告
|
logg.WriteLog("四川省中标公告 开始获取", "四川");
|
while (true)
|
{
|
logg.WriteLog(currPage.ToString(), "四川");
|
string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=00102®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
|
using (HttpClient client = new HttpClient())
|
{
|
client.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage response = client.GetAsync(address).Result;
|
var res = response.Content.ReadAsStringAsync().Result;
|
|
sichuanjieshou sichuan = JsonConvert.DeserializeObject<sichuanjieshou>(res);
|
|
//保存项目到sqlserveer和elasticsearch
|
if (sichuan.data != null && sichuan.data.Count > 0)
|
{
|
foreach (var sichuanjieshoudtl in sichuan.data)
|
{
|
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
|
var notimestart = DateTime.Parse(notime);
|
var notimeend = notimestart.AddDays(1);
|
var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "03" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
|
if (zhengfuProjectCount <= 0)
|
{
|
var zhengfuProject = new ZhengfuProject();
|
zhengfuProject.Id = Guid.NewGuid().ToString();
|
zhengfuProject.Gglx = "03";
|
string gglxName = "结果公告";
|
zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime;
|
zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime;
|
zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode;
|
string regionName = sichuanjieshoudtl.regionName;
|
zhengfuProject.Sheng = "510000";
|
string ShengName = "四川省";
|
string CityName = "";
|
if ("510001" == sichuanjieshoudtl.regionCode)
|
{
|
zhengfuProject.RegionCode = "";
|
regionName = "";
|
zhengfuProject.City = "510001";
|
CityName = "省本级";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5101"))
|
{
|
zhengfuProject.City = "510100";
|
CityName = "成都市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5103"))
|
{
|
zhengfuProject.City = "510300";
|
CityName = "自贡市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5104"))
|
{
|
zhengfuProject.City = "510400";
|
CityName = "攀枝花市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5105"))
|
{
|
zhengfuProject.City = "510500";
|
CityName = "泸州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5106"))
|
{
|
zhengfuProject.City = "510600";
|
CityName = "德阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5107"))
|
{
|
zhengfuProject.City = "510700";
|
CityName = "绵阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5108"))
|
{
|
zhengfuProject.City = "510800";
|
CityName = "广元市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5109"))
|
{
|
zhengfuProject.City = "510900";
|
CityName = "遂宁市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5110"))
|
{
|
zhengfuProject.City = "511000";
|
CityName = "内江市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5111"))
|
{
|
zhengfuProject.City = "511100";
|
CityName = "乐山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5113"))
|
{
|
zhengfuProject.City = "511300";
|
CityName = "南充市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5114"))
|
{
|
zhengfuProject.City = "511400";
|
CityName = "眉山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5115"))
|
{
|
zhengfuProject.City = "511500";
|
CityName = "宜宾市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5116"))
|
{
|
zhengfuProject.City = "511600";
|
CityName = "广安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5117"))
|
{
|
zhengfuProject.City = "511700";
|
CityName = "达州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5118"))
|
{
|
zhengfuProject.City = "511800";
|
CityName = "雅安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5119"))
|
{
|
zhengfuProject.City = "511900";
|
CityName = "巴中市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5120"))
|
{
|
zhengfuProject.City = "512000";
|
CityName = "资阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5132"))
|
{
|
zhengfuProject.City = "513200";
|
CityName = "阿坝州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5133"))
|
{
|
zhengfuProject.City = "513300";
|
CityName = "甘孜州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5134"))
|
{
|
zhengfuProject.City = "513400";
|
CityName = "凉山州";
|
}
|
|
|
string cgfsName = "";
|
string BieCgfs = "";
|
switch (sichuanjieshoudtl.purchaseManner)
|
{
|
case "1":
|
zhengfuProject.Cgfs = "1";
|
cgfsName = "公开招标";
|
BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c11";
|
break;
|
case "2":
|
zhengfuProject.Cgfs = "2";
|
cgfsName = "邀请招标";
|
BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c12";
|
break;
|
case "3":
|
zhengfuProject.Cgfs = "5";
|
cgfsName = "竞争性谈判";
|
BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c14";
|
break;
|
case "4":
|
zhengfuProject.Cgfs = "4";
|
cgfsName = "询价";
|
BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c15";
|
break;
|
case "5":
|
zhengfuProject.Cgfs = "7";
|
cgfsName = "单一来源";
|
BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c16";
|
break;
|
case "6":
|
zhengfuProject.Cgfs = "3";
|
cgfsName = "竞争性磋商";
|
BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c13";
|
break;
|
case "9":
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
BieCgfs = "";
|
break;
|
default:
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
BieCgfs = "";
|
break;
|
}
|
zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
|
zhengfuProject.Title = sichuanjieshoudtl.title;
|
zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle;
|
zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser;
|
|
if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget))
|
{
|
zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget);
|
}
|
zhengfuProject.Agency = sichuanjieshoudtl.agency;
|
zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode;
|
zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty);
|
zhengfuProject.RecStatus = "A";
|
zhengfuProject.Creater = "1";
|
zhengfuProject.Createtime = DateTime.Now;
|
zhengfuProject.Modifier = "1";
|
zhengfuProject.Modifytime = zhengfuProject.Createtime;
|
_ccontext.ZhengfuProjects.Add(zhengfuProject);
|
|
|
Project project = new Project();
|
project.Id = zhengfuProject.Id;
|
project.StartTime = zhengfuProject.NoticeTime;
|
project.Sheng = zhengfuProject.Sheng;
|
project.City = zhengfuProject.City;
|
if (project.City == "510001")
|
{
|
project.AreaId = "510002";
|
}
|
else
|
{
|
project.AreaId = zhengfuProject.RegionCode;
|
}
|
|
project.Cgfs = BieCgfs;
|
project.Number = zhengfuProject.OpenTenderCode;
|
project.Name = zhengfuProject.Title;
|
|
if (zhengfuProject.Budget.HasValue)
|
{
|
project.Caigouyusuan = zhengfuProject.Budget.Value / new decimal(10000);
|
}
|
project.Question = zhengfuProject.Pageurl;
|
var parser = new HtmlParser();
|
var document = parser.ParseDocument(zhengfuProject.Content);
|
var cDocumet = document.All.Where(m => m.Id == "noticeArea").FirstOrDefault();
|
if (cDocumet != null)
|
{
|
//var cDocumet = contentList.First();
|
var Numberss = cDocumet.QuerySelectorAll("p");
|
foreach (var Number in Numberss)
|
{
|
|
if (Number != null && Number.TextContent.Contains("废标理由"))
|
{
|
var Fbyy = Number.QuerySelector("span");
|
project.Fbyy = Fbyy.TextContent.Trim();
|
}
|
}
|
|
var Caigourennames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgName");
|
if (Caigourennames != null && Caigourennames.ToList().Count > 0)
|
{
|
var Caigourenname = Caigourennames.First();
|
project.Caigourenname = Caigourenname.TextContent;
|
}
|
var Caigourenaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgAddress");
|
if (Caigourenaddresss != null && Caigourenaddresss.ToList().Count > 0)
|
{
|
var Caigourenaddress = Caigourenaddresss.First();
|
project.Caigourenaddress = Caigourenaddress.TextContent;
|
}
|
var Caigourencontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserLinkTel");
|
if (Caigourencontacts != null && Caigourencontacts.ToList().Count > 0)
|
{
|
var Caigourencontact = Caigourencontacts.First();
|
project.Caigourencontact = Caigourencontact.TextContent;
|
}
|
var Dailinames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agencyName");
|
if (Dailinames != null && Dailinames.ToList().Count > 0)
|
{
|
var Dailiname = Dailinames.First();
|
project.Dailiname = Dailiname.TextContent;
|
}
|
var Dailiaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentAddress");
|
if (Dailiaddresss != null && Dailiaddresss.ToList().Count > 0)
|
{
|
var Dailiaddress = Dailiaddresss.First();
|
project.Dailiaddress = Dailiaddress.TextContent;
|
}
|
var Dailicontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentLinkTel");
|
if (Dailicontacts != null && Dailicontacts.ToList().Count > 0)
|
{
|
var Dailicontact = Dailicontacts.First();
|
project.Dailicontact = Dailicontact.TextContent;
|
}
|
|
|
|
var zhuanjias = cDocumet.QuerySelectorAll("div").Where(m => m.ClassName == "noticeBidResult-noticeBidResult _notice_content_noticeBidResult-noticeBidResult expert dynamic-form-editor");
|
if (zhuanjias != null && zhuanjias.ToList().Count > 0)
|
{
|
var zhuanjia = zhuanjias.First();
|
|
project.Zhuanjia = zhuanjia.TextContent.Trim();//OuterHtml
|
}
|
}
|
|
|
|
|
|
|
project.Creater = "111";
|
project.Createtime = zhengfuProject.Createtime;
|
project.Modifier = project.Creater;
|
project.Modifytime = project.Createtime;
|
project.RecStatus = "A";
|
|
_ccontext1.Projects.Add(project);
|
if (cDocumet != null)
|
{
|
//var cDocumet = contentList.First();
|
var table1 = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_noticeBidResult-noticeBidResult");
|
|
if (table1 != null && table1.ToList().Count > 0)
|
{
|
var table = table1.First().QuerySelectorAll("table");
|
if (table != null && table.ToList().Count > 0)
|
{
|
for (int i = 0; i < table.Length; i++)
|
{
|
var td = table[i].QuerySelectorAll("td");
|
if (td.Length >= 3)
|
{
|
|
var projectzhongbiao = new Projectzhongbiao();
|
projectzhongbiao.ZhongbiaoId = Guid.NewGuid().ToString();
|
projectzhongbiao.ProjectId = project.Id;
|
if ( !string.IsNullOrWhiteSpace(td[0].TextContent) && td[0].TextContent.Trim().Length > 500)
|
{
|
projectzhongbiao.Names = td[0].TextContent.Trim().Substring(0,500);
|
}
|
else
|
{
|
projectzhongbiao.Names = td[0].TextContent.Trim();
|
}
|
|
projectzhongbiao.Price = td[2].TextContent.Trim();
|
_ccontext1.Projectzhongbiaos.Add(projectzhongbiao);
|
}
|
}
|
}
|
|
}
|
}
|
|
|
|
|
|
|
|
|
|
|
|
string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
|
string result = string.Empty;
|
Uri postUrl = new Uri(url);
|
eswebcrawler eswebcrawler1 = new eswebcrawler();
|
eswebcrawler1.Id = zhengfuProject.Id;
|
eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss");
|
eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null;
|
eswebcrawler1.sheng = zhengfuProject.Sheng;
|
eswebcrawler1.shengName = ShengName;
|
eswebcrawler1.city = zhengfuProject.City;
|
eswebcrawler1.cityName = CityName;
|
eswebcrawler1.regionCode = zhengfuProject.RegionCode;
|
eswebcrawler1.regionName = regionName;
|
eswebcrawler1.cgfs = zhengfuProject.Cgfs;
|
eswebcrawler1.cgfsName = cgfsName;
|
eswebcrawler1.gglx = zhengfuProject.Gglx;
|
eswebcrawler1.gglxName = gglxName;
|
eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode;
|
eswebcrawler1.title = zhengfuProject.Title;
|
eswebcrawler1.shorttitle = zhengfuProject.Shorttitle;
|
eswebcrawler1.pageurl = zhengfuProject.Pageurl;
|
eswebcrawler1.pingmu = zhengfuProject.Pingmu;
|
eswebcrawler1.pingmuName = "";
|
eswebcrawler1.purchaser = zhengfuProject.Purchaser;
|
eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : "";
|
eswebcrawler1.agency = zhengfuProject.Agency;
|
eswebcrawler1.agencyCode = zhengfuProject.AgencyCode;
|
eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss");
|
|
string requestJson = JsonConvert.SerializeObject(eswebcrawler1);
|
|
using (HttpContent httpContent = new StringContent(requestJson))
|
{
|
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
|
using (HttpClient httpClient = new HttpClient())
|
{
|
httpClient.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result;
|
result = responseMessage.Content.ReadAsStringAsync().Result;
|
}
|
|
}
|
|
Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result);
|
|
if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1")
|
{
|
|
}
|
else
|
{
|
throw new Exception("保存数据库出错！");
|
}
|
_ccontext1.SaveChanges();
|
|
_ccontext.SaveChanges();
|
}
|
}
|
}
|
|
|
if (currPage * pageSize >= sichuan.total)
|
{
|
break;
|
}
|
else
|
{
|
currPage += 1;
|
}
|
Thread.CurrentThread.Join(1000 * 20);//阻止设定时间
|
}
|
|
|
}
|
logg.WriteLog("四川省中标公告 获取完成", "四川");
|
|
//翻页信息
|
currPage = 1;
|
pageSize = 10;
|
//开始访问 采购公告
|
logg.WriteLog("四川省更正公告 开始获取", "四川");
|
while (true)
|
{
|
logg.WriteLog(currPage.ToString(), "四川");
|
string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=00103®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
|
using (HttpClient client = new HttpClient())
|
{
|
client.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage response = client.GetAsync(address).Result;
|
var res = response.Content.ReadAsStringAsync().Result;
|
|
sichuanjieshou sichuan = JsonConvert.DeserializeObject<sichuanjieshou>(res);
|
|
//保存项目到sqlserveer和elasticsearch
|
if (sichuan.data != null && sichuan.data.Count > 0)
|
{
|
foreach (var sichuanjieshoudtl in sichuan.data)
|
{
|
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
|
var notimestart = DateTime.Parse(notime);
|
var notimeend = notimestart.AddDays(1);
|
var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "04" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
|
if (zhengfuProjectCount <= 0)
|
{
|
var zhengfuProject = new ZhengfuProject();
|
zhengfuProject.Id = Guid.NewGuid().ToString();
|
zhengfuProject.Gglx = "04";
|
string gglxName = "更正公告";
|
zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime;
|
zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime;
|
zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode;
|
string regionName = sichuanjieshoudtl.regionName;
|
zhengfuProject.Sheng = "510000";
|
string ShengName = "四川省";
|
string CityName = "";
|
if ("510001" == sichuanjieshoudtl.regionCode)
|
{
|
zhengfuProject.RegionCode = "";
|
regionName = "";
|
zhengfuProject.City = "510001";
|
CityName = "省本级";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5101"))
|
{
|
zhengfuProject.City = "510100";
|
CityName = "成都市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5103"))
|
{
|
zhengfuProject.City = "510300";
|
CityName = "自贡市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5104"))
|
{
|
zhengfuProject.City = "510400";
|
CityName = "攀枝花市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5105"))
|
{
|
zhengfuProject.City = "510500";
|
CityName = "泸州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5106"))
|
{
|
zhengfuProject.City = "510600";
|
CityName = "德阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5107"))
|
{
|
zhengfuProject.City = "510700";
|
CityName = "绵阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5108"))
|
{
|
zhengfuProject.City = "510800";
|
CityName = "广元市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5109"))
|
{
|
zhengfuProject.City = "510900";
|
CityName = "遂宁市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5110"))
|
{
|
zhengfuProject.City = "511000";
|
CityName = "内江市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5111"))
|
{
|
zhengfuProject.City = "511100";
|
CityName = "乐山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5113"))
|
{
|
zhengfuProject.City = "511300";
|
CityName = "南充市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5114"))
|
{
|
zhengfuProject.City = "5114";
|
CityName = "眉山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5115"))
|
{
|
zhengfuProject.City = "511500";
|
CityName = "宜宾市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5116"))
|
{
|
zhengfuProject.City = "511600";
|
CityName = "广安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5117"))
|
{
|
zhengfuProject.City = "511700";
|
CityName = "达州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5118"))
|
{
|
zhengfuProject.City = "511800";
|
CityName = "雅安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5119"))
|
{
|
zhengfuProject.City = "511900";
|
CityName = "巴中市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5120"))
|
{
|
zhengfuProject.City = "512000";
|
CityName = "资阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5132"))
|
{
|
zhengfuProject.City = "513200";
|
CityName = "阿坝州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5133"))
|
{
|
zhengfuProject.City = "513300";
|
CityName = "甘孜州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5134"))
|
{
|
zhengfuProject.City = "513400";
|
CityName = "凉山州";
|
}
|
|
|
string cgfsName = "";
|
switch (sichuanjieshoudtl.purchaseManner)
|
{
|
case "1":
|
zhengfuProject.Cgfs = "1";
|
cgfsName = "公开招标";
|
break;
|
case "2":
|
zhengfuProject.Cgfs = "2";
|
cgfsName = "邀请招标";
|
break;
|
case "3":
|
zhengfuProject.Cgfs = "5";
|
cgfsName = "竞争性谈判";
|
break;
|
case "4":
|
zhengfuProject.Cgfs = "4";
|
cgfsName = "询价";
|
break;
|
case "5":
|
zhengfuProject.Cgfs = "7";
|
cgfsName = "单一来源";
|
break;
|
case "6":
|
zhengfuProject.Cgfs = "3";
|
cgfsName = "竞争性磋商";
|
break;
|
case "9":
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
break;
|
default:
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
break;
|
}
|
zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
|
zhengfuProject.Title = sichuanjieshoudtl.title;
|
zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle;
|
zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser;
|
|
if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget))
|
{
|
zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget);
|
}
|
zhengfuProject.Agency = sichuanjieshoudtl.agency;
|
zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode;
|
zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty);
|
zhengfuProject.RecStatus = "A";
|
zhengfuProject.Creater = "1";
|
zhengfuProject.Createtime = DateTime.Now;
|
zhengfuProject.Modifier = "1";
|
zhengfuProject.Modifytime = zhengfuProject.Createtime;
|
_ccontext.ZhengfuProjects.Add(zhengfuProject);
|
|
|
string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
|
string result = string.Empty;
|
Uri postUrl = new Uri(url);
|
eswebcrawler eswebcrawler1 = new eswebcrawler();
|
eswebcrawler1.Id = zhengfuProject.Id;
|
eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss");
|
eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null;
|
eswebcrawler1.sheng = zhengfuProject.Sheng;
|
eswebcrawler1.shengName = ShengName;
|
eswebcrawler1.city = zhengfuProject.City;
|
eswebcrawler1.cityName = CityName;
|
eswebcrawler1.regionCode = zhengfuProject.RegionCode;
|
eswebcrawler1.regionName = regionName;
|
eswebcrawler1.cgfs = zhengfuProject.Cgfs;
|
eswebcrawler1.cgfsName = cgfsName;
|
eswebcrawler1.gglx = zhengfuProject.Gglx;
|
eswebcrawler1.gglxName = gglxName;
|
eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode;
|
eswebcrawler1.title = zhengfuProject.Title;
|
eswebcrawler1.shorttitle = zhengfuProject.Shorttitle;
|
eswebcrawler1.pageurl = zhengfuProject.Pageurl;
|
eswebcrawler1.pingmu = zhengfuProject.Pingmu;
|
eswebcrawler1.pingmuName = "";
|
eswebcrawler1.purchaser = zhengfuProject.Purchaser;
|
eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : "";
|
eswebcrawler1.agency = zhengfuProject.Agency;
|
eswebcrawler1.agencyCode = zhengfuProject.AgencyCode;
|
eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss");
|
|
string requestJson = JsonConvert.SerializeObject(eswebcrawler1);
|
|
using (HttpContent httpContent = new StringContent(requestJson))
|
{
|
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
|
using (HttpClient httpClient = new HttpClient())
|
{
|
httpClient.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result;
|
result = responseMessage.Content.ReadAsStringAsync().Result;
|
}
|
|
}
|
|
Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result);
|
|
if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1")
|
{
|
|
}
|
else
|
{
|
throw new Exception("保存数据库出错！");
|
}
|
|
|
_ccontext.SaveChanges();
|
}
|
}
|
}
|
|
|
if (currPage * pageSize >= sichuan.total)
|
{
|
break;
|
}
|
else
|
{
|
currPage += 1;
|
}
|
Thread.CurrentThread.Join(1000 * 20);//阻止设定时间
|
}
|
|
|
}
|
logg.WriteLog("四川省更正公告 获取完成", "四川");
|
|
|
//翻页信息
|
currPage = 1;
|
pageSize = 10;
|
//开始访问 采购公告
|
logg.WriteLog("四川省废标公告 开始获取", "四川");
|
while (true)
|
{
|
logg.WriteLog(currPage.ToString(), "四川");
|
string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "¬iceType=001004,001006®ionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
|
using (HttpClient client = new HttpClient())
|
{
|
client.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage response = client.GetAsync(address).Result;
|
var res = response.Content.ReadAsStringAsync().Result;
|
|
sichuanjieshou sichuan = JsonConvert.DeserializeObject<sichuanjieshou>(res);
|
|
//保存项目到sqlserveer和elasticsearch
|
if (sichuan.data != null && sichuan.data.Count > 0)
|
{
|
foreach (var sichuanjieshoudtl in sichuan.data)
|
{
|
var pageurl1 = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
var notime = sichuanjieshoudtl.noticeTime.ToString("yyyy-MM-dd");
|
var notimestart = DateTime.Parse(notime);
|
var notimeend = notimestart.AddDays(1);
|
|
var zhengfuProjectCount = _ccontext.ZhengfuProjects.Count(x => (x.RecStatus == "A" && x.Pageurl == pageurl1) || (x.RecStatus == "A" && x.Title == sichuanjieshoudtl.title && x.Gglx == "11" && x.NoticeTime >= notimestart && x.NoticeTime < notimeend));
|
if (zhengfuProjectCount <= 0)
|
{
|
var zhengfuProject = new ZhengfuProject();
|
zhengfuProject.Id = Guid.NewGuid().ToString();
|
zhengfuProject.Gglx = "11";
|
string gglxName = "废标公告";
|
zhengfuProject.NoticeTime = sichuanjieshoudtl.noticeTime;
|
zhengfuProject.OpenTenderTime = sichuanjieshoudtl.openTenderTime;
|
zhengfuProject.RegionCode = sichuanjieshoudtl.regionCode;
|
string regionName = sichuanjieshoudtl.regionName;
|
zhengfuProject.Sheng = "510000";
|
string ShengName = "四川省";
|
string CityName = "";
|
if ("510001" == sichuanjieshoudtl.regionCode)
|
{
|
zhengfuProject.RegionCode = "";
|
regionName = "";
|
zhengfuProject.City = "510001";
|
CityName = "省本级";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5101"))
|
{
|
zhengfuProject.City = "510100";
|
CityName = "成都市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5103"))
|
{
|
zhengfuProject.City = "510300";
|
CityName = "自贡市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5104"))
|
{
|
zhengfuProject.City = "510400";
|
CityName = "攀枝花市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5105"))
|
{
|
zhengfuProject.City = "510500";
|
CityName = "泸州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5106"))
|
{
|
zhengfuProject.City = "510600";
|
CityName = "德阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5107"))
|
{
|
zhengfuProject.City = "510700";
|
CityName = "绵阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5108"))
|
{
|
zhengfuProject.City = "510800";
|
CityName = "广元市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5109"))
|
{
|
zhengfuProject.City = "510900";
|
CityName = "遂宁市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5110"))
|
{
|
zhengfuProject.City = "511000";
|
CityName = "内江市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5111"))
|
{
|
zhengfuProject.City = "511100";
|
CityName = "乐山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5113"))
|
{
|
zhengfuProject.City = "511300";
|
CityName = "南充市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5114"))
|
{
|
zhengfuProject.City = "5114";
|
CityName = "眉山市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5115"))
|
{
|
zhengfuProject.City = "511500";
|
CityName = "宜宾市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5116"))
|
{
|
zhengfuProject.City = "511600";
|
CityName = "广安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5117"))
|
{
|
zhengfuProject.City = "511700";
|
CityName = "达州市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5118"))
|
{
|
zhengfuProject.City = "511800";
|
CityName = "雅安市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5119"))
|
{
|
zhengfuProject.City = "511900";
|
CityName = "巴中市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5120"))
|
{
|
zhengfuProject.City = "512000";
|
CityName = "资阳市";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5132"))
|
{
|
zhengfuProject.City = "513200";
|
CityName = "阿坝州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5133"))
|
{
|
zhengfuProject.City = "513300";
|
CityName = "甘孜州";
|
}
|
else if (sichuanjieshoudtl.regionCode.StartsWith("5134"))
|
{
|
zhengfuProject.City = "513400";
|
CityName = "凉山州";
|
}
|
|
|
string cgfsName = "";
|
switch (sichuanjieshoudtl.purchaseManner)
|
{
|
case "1":
|
zhengfuProject.Cgfs = "1";
|
cgfsName = "公开招标";
|
break;
|
case "2":
|
zhengfuProject.Cgfs = "2";
|
cgfsName = "邀请招标";
|
break;
|
case "3":
|
zhengfuProject.Cgfs = "5";
|
cgfsName = "竞争性谈判";
|
break;
|
case "4":
|
zhengfuProject.Cgfs = "4";
|
cgfsName = "询价";
|
break;
|
case "5":
|
zhengfuProject.Cgfs = "7";
|
cgfsName = "单一来源";
|
break;
|
case "6":
|
zhengfuProject.Cgfs = "3";
|
cgfsName = "竞争性磋商";
|
break;
|
case "9":
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
break;
|
default:
|
zhengfuProject.Cgfs = "10";
|
cgfsName = "其他";
|
break;
|
}
|
zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
|
zhengfuProject.Title = sichuanjieshoudtl.title;
|
zhengfuProject.Shorttitle = sichuanjieshoudtl.shorttitle;
|
zhengfuProject.Pageurl = sichuanpageurll + sichuanjieshoudtl.pageurl;
|
zhengfuProject.Purchaser = sichuanjieshoudtl.purchaser;
|
|
if (!string.IsNullOrWhiteSpace(sichuanjieshoudtl.budget))
|
{
|
zhengfuProject.Budget = decimal.Parse(sichuanjieshoudtl.budget);
|
}
|
zhengfuProject.Agency = sichuanjieshoudtl.agency;
|
zhengfuProject.AgencyCode = sichuanjieshoudtl.agencyCode;
|
zhengfuProject.Content = sichuanjieshoudtl.content.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("\t", string.Empty).Replace(@"\", string.Empty);
|
zhengfuProject.RecStatus = "A";
|
zhengfuProject.Creater = "1";
|
zhengfuProject.Createtime = DateTime.Now;
|
zhengfuProject.Modifier = "1";
|
zhengfuProject.Modifytime = zhengfuProject.Createtime;
|
_ccontext.ZhengfuProjects.Add(zhengfuProject);
|
|
|
var projects = _ccontext1.Projects.Where(x => x.RecStatus == "A" && x.Number == zhengfuProject.OpenTenderCode && x.Fblx != null ).ToList();
|
if (projects.Count > 0)
|
{
|
foreach (var project in projects)
|
{
|
project.Fblx = "停止评审";
|
}
|
|
var parser = new HtmlParser();
|
var document = parser.ParseDocument(zhengfuProject.Content);
|
var contentList = document.All.Where(m => m.Id == "noticeArea");
|
var cDocumet = contentList.First();
|
var Numberss = cDocumet.QuerySelectorAll("p");
|
foreach (var Number in Numberss)
|
{
|
|
if (Number != null && Number.TextContent.Contains("废标理由"))
|
{
|
if (Number.TextContent.IndexOf("span") >= 0)
|
{
|
var Fbyy = Number.QuerySelector("span");
|
foreach (var project in projects)
|
{
|
project.Fbyy = Fbyy.TextContent.Trim();
|
project.Fblx = "停止评审";
|
}
|
}
|
|
|
}
|
if (Number != null && Number.TextContent.Contains("终止原因"))
|
{
|
if(Number.TextContent.IndexOf("span") >= 0)
|
{
|
var Fbyy = Number.QuerySelector("span");
|
foreach (var project in projects)
|
{
|
project.Fbyy = Fbyy.TextContent.Trim();
|
project.Fblx = "停止评审";
|
}
|
}
|
|
|
}
|
}
|
var feibiaoliyous = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_reason-reason");
|
if (feibiaoliyous != null && feibiaoliyous.ToList().Count > 0)
|
{
|
var feibiaoliyou = feibiaoliyous.First();
|
foreach (var project in projects)
|
{
|
project.Fbyy = feibiaoliyou.TextContent.Replace("\n", "").Replace("\t", "");
|
project.Fblx = "终止采购";
|
}
|
}
|
|
_ccontext1.SaveChanges();
|
}
|
|
|
|
|
|
|
string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
|
string result = string.Empty;
|
Uri postUrl = new Uri(url);
|
eswebcrawler eswebcrawler1 = new eswebcrawler();
|
eswebcrawler1.Id = zhengfuProject.Id;
|
eswebcrawler1.noticeTime = zhengfuProject.NoticeTime.ToString("yyyy-MM-dd HH:mm:ss");
|
eswebcrawler1.openTenderTime = zhengfuProject.OpenTenderTime.HasValue ? zhengfuProject.OpenTenderTime.Value.ToString("yyyy-MM-dd HH:mm:ss") : null;
|
eswebcrawler1.sheng = zhengfuProject.Sheng;
|
eswebcrawler1.shengName = ShengName;
|
eswebcrawler1.city = zhengfuProject.City;
|
eswebcrawler1.cityName = CityName;
|
eswebcrawler1.regionCode = zhengfuProject.RegionCode;
|
eswebcrawler1.regionName = regionName;
|
eswebcrawler1.cgfs = zhengfuProject.Cgfs;
|
eswebcrawler1.cgfsName = cgfsName;
|
eswebcrawler1.gglx = zhengfuProject.Gglx;
|
eswebcrawler1.gglxName = gglxName;
|
eswebcrawler1.openTenderCode = zhengfuProject.OpenTenderCode;
|
eswebcrawler1.title = zhengfuProject.Title;
|
eswebcrawler1.shorttitle = zhengfuProject.Shorttitle;
|
eswebcrawler1.pageurl = zhengfuProject.Pageurl;
|
eswebcrawler1.pingmu = zhengfuProject.Pingmu;
|
eswebcrawler1.pingmuName = "";
|
eswebcrawler1.purchaser = zhengfuProject.Purchaser;
|
eswebcrawler1.budget = zhengfuProject.Budget.HasValue ? zhengfuProject.Budget.Value.ToString() : "";
|
eswebcrawler1.agency = zhengfuProject.Agency;
|
eswebcrawler1.agencyCode = zhengfuProject.AgencyCode;
|
eswebcrawler1.modifyTime = zhengfuProject.Modifytime.ToString("yyyy-MM-dd HH:mm:ss");
|
|
string requestJson = JsonConvert.SerializeObject(eswebcrawler1);
|
|
using (HttpContent httpContent = new StringContent(requestJson))
|
{
|
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
|
using (HttpClient httpClient = new HttpClient())
|
{
|
httpClient.Timeout = TimeSpan.FromSeconds(60);
|
HttpResponseMessage responseMessage = httpClient.PutAsync(postUrl, httpContent).Result;
|
result = responseMessage.Content.ReadAsStringAsync().Result;
|
}
|
|
}
|
|
Newtonsoft.Json.Linq.JObject jobject = (Newtonsoft.Json.Linq.JObject)JsonConvert.DeserializeObject(result);
|
|
if (jobject["error"] == null && jobject["_shards"]["successful"].ToString() == "1")
|
{
|
|
}
|
else
|
{
|
throw new Exception("保存数据库出错！");
|
}
|
|
|
_ccontext.SaveChanges();
|
}
|
}
|
}
|
|
|
if (currPage * pageSize >= sichuan.total)
|
{
|
break;
|
}
|
else
|
{
|
currPage += 1;
|
}
|
Thread.CurrentThread.Join(1000 * 20);//阻止设定时间
|
}
|
|
|
}
|
logg.WriteLog("四川省废标公告 获取完成", "四川");
|
|
|
|
//保存最后更新时间
|
if (updatetime != null)
|
{
|
updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
|
_ccontext.SaveChanges();
|
}
|
else
|
{
|
updatetime = new Updatetime();
|
updatetime.Id = Guid.NewGuid().ToString();
|
updatetime.Sheng = "四川省";
|
updatetime.Updatetime1 = DateTime.Now.AddDays(-1);
|
_ccontext.Updatetimes.Add(updatetime);
|
_ccontext.SaveChanges();
|
}
|
}
|
catch (Exception ex)
|
{
|
logg.WriteLog(ex.Message, "四川");
|
|
logg.WriteLog(ex.StackTrace, "四川");
|
|
logg.WriteLog(ex.ToString(), "四川");
|
}
|
finally
|
{
|
|
}
|
}
|
}
|
}
|
