username@email.com
2022-11-24 e1064dd08bba4326d19c81a0b9f4da18c2fa462d
zhengcaioa/Crawler/sichuan/sichuanoperation.cs
@@ -1,4 +1,6 @@
using Newtonsoft.Json;
using AngleSharp.Html.Parser;
using DTO;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -12,10 +14,9 @@
{
   public class sichuanoperation
    {
        //public static string api_domain = "http://192.168.0.116:9200";
        public static string api_domain = "http://localhost:9200";
        public static string sichuanpageurll = "https://zfcg.scsczt.cn";
        public static void operations(WebCrawlerContext _ccontext)
        public static void operations(WebCrawlerContext _ccontext, zhengcaioaContext _ccontext1)
        {
            try
            {
@@ -43,10 +44,10 @@
                int currPage = 1;
                int pageSize = 10;
                //开始访问 采购公告
                logg.WriteLog("四川省采购公告 开始获取");
                logg.WriteLog("四川省采购公告 开始获取", "四川");
                while (true)
                {
                    logg.WriteLog(currPage.ToString());
                    logg.WriteLog(currPage.ToString(), "四川");
                    string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "&noticeType=00101&regionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
                    using (HttpClient client = new HttpClient())
                    {
@@ -221,6 +222,10 @@
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            break;
                                        default:
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            break;
                                    }
                                    zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
                                    zhengfuProject.Title = sichuanjieshoudtl.title;
@@ -243,7 +248,7 @@
                                    _ccontext.ZhengfuProjects.Add(zhengfuProject);
                                    string url = $"{api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    string result = string.Empty;
                                    Uri postUrl = new Uri(url);
                                    eswebcrawler eswebcrawler1 = new eswebcrawler();
@@ -317,16 +322,16 @@
                }
                logg.WriteLog("四川省采购公告 获取完成");
                logg.WriteLog("四川省采购公告 获取完成","四川");
                //翻页信息
                currPage = 1;
                pageSize = 10;
                //开始访问 采购公告
                logg.WriteLog("四川省中标公告 开始获取");
                logg.WriteLog("四川省中标公告 开始获取", "四川");
                while (true)
                {
                    logg.WriteLog(currPage.ToString());
                    logg.WriteLog(currPage.ToString(), "四川");
                    string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "&noticeType=00102&regionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
                    using (HttpClient client = new HttpClient())
                    {
@@ -420,7 +425,7 @@
                                    }
                                    else if (sichuanjieshoudtl.regionCode.StartsWith("5114"))
                                    {
                                        zhengfuProject.City = "5114";
                                        zhengfuProject.City = "511400";
                                        CityName = "眉山市";
                                    }
                                    else if (sichuanjieshoudtl.regionCode.StartsWith("5115"))
@@ -471,35 +476,48 @@
                                    string cgfsName = "";
                                    string BieCgfs = "";
                                    switch (sichuanjieshoudtl.purchaseManner)
                                    {
                                        case "1":
                                            zhengfuProject.Cgfs = "1";
                                            cgfsName = "公开招标";
                                            BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c11";
                                            break;
                                        case "2":
                                            zhengfuProject.Cgfs = "2";
                                            cgfsName = "邀请招标";
                                            BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c12";
                                            break;
                                        case "3":
                                            zhengfuProject.Cgfs = "5";
                                            cgfsName = "竞争性谈判";
                                            BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c14";
                                            break;
                                        case "4":
                                            zhengfuProject.Cgfs = "4";
                                            cgfsName = "询价";
                                            BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c15";
                                            break;
                                        case "5":
                                            zhengfuProject.Cgfs = "7";
                                            cgfsName = "单一来源";
                                            BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c16";
                                            break;
                                        case "6":
                                            zhengfuProject.Cgfs = "3";
                                            cgfsName = "竞争性磋商";
                                            BieCgfs = "e4c93811-b9b1-4998-89f5-c416ebab0c13";
                                            break;
                                        case "9":
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            BieCgfs = "";
                                            break;
                                        default:
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            BieCgfs = "";
                                            break;
                                    }
                                    zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
@@ -523,7 +541,132 @@
                                    _ccontext.ZhengfuProjects.Add(zhengfuProject);
                                    string url = $"{api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    Project project = new Project();
                                    project.Id = zhengfuProject.Id;
                                    project.StartTime = zhengfuProject.NoticeTime;
                                    project.Sheng = zhengfuProject.Sheng;
                                    project.City = zhengfuProject.City;
                                    if(project.City== "510001")
                                    {
                                        project.AreaId  = "510002";
                                    }
                                    else
                                    {
                                        project.AreaId = zhengfuProject.RegionCode;
                                    }
                                    project.Cgfs = BieCgfs;
                                    project.Number = zhengfuProject.OpenTenderCode;
                                    project.Name = zhengfuProject.Title;
                                    if (zhengfuProject.Budget.HasValue)
                                    {
                                        project.Caigouyusuan = zhengfuProject.Budget.Value / new decimal(10000);
                                    }
                                    project.Question = zhengfuProject.Pageurl;
                                    var parser = new HtmlParser();
                                    var document = parser.ParseDocument(zhengfuProject.Content);
                                    var contentList = document.All.Where(m => m.Id == "noticeArea");
                                    var cDocumet = contentList.First();
                                    var Numberss = cDocumet.QuerySelectorAll("p");
                                    foreach (var Number in Numberss)
                                    {
                                        if (Number != null && Number.TextContent.Contains("废标理由"))
                                        {
                                            var Fbyy = Number.QuerySelector("span");
                                            project.Fbyy = Fbyy.TextContent.Trim();
                                        }
                                    }
                                    var Caigourennames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgName");
                                    if (Caigourennames != null && Caigourennames.ToList().Count > 0)
                                    {
                                        var Caigourenname = Caigourennames.First();
                                        project.Caigourenname = Caigourenname.TextContent;
                                    }
                                    var Caigourenaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserOrgAddress");
                                    if (Caigourenaddresss != null && Caigourenaddresss.ToList().Count > 0)
                                    {
                                        var Caigourenaddress = Caigourenaddresss.First();
                                        project.Caigourenaddress = Caigourenaddress.TextContent;
                                    }
                                    var Caigourencontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticePurchase-purchaserLinkTel");
                                    if (Caigourencontacts != null && Caigourencontacts.ToList().Count > 0)
                                    {
                                        var Caigourencontact = Caigourencontacts.First();
                                        project.Caigourencontact = Caigourencontact.TextContent;
                                    }
                                    var Dailinames = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agencyName");
                                    if (Dailinames != null && Dailinames.ToList().Count > 0)
                                    {
                                        var Dailiname = Dailinames.First();
                                        project.Dailiname = Dailiname.TextContent;
                                    }
                                    var Dailiaddresss = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentAddress");
                                    if (Dailiaddresss != null && Dailiaddresss.ToList().Count > 0)
                                    {
                                        var Dailiaddress = Dailiaddresss.First();
                                        project.Dailiaddress = Dailiaddress.TextContent;
                                    }
                                    var Dailicontacts = cDocumet.QuerySelectorAll("span").Where(m => m.Id == "_notice_content_noticeAgency-agentLinkTel");
                                    if (Dailicontacts != null && Dailicontacts.ToList().Count > 0)
                                    {
                                        var Dailicontact = Dailicontacts.First();
                                        project.Dailicontact = Dailicontact.TextContent;
                                    }
                                    var zhuanjias = cDocumet.QuerySelectorAll("div").Where(m => m.ClassName == "noticeBidResult-noticeBidResult _notice_content_noticeBidResult-noticeBidResult expert dynamic-form-editor");
                                    if (zhuanjias != null && zhuanjias.ToList().Count > 0)
                                    {
                                        var zhuanjia = zhuanjias.First();
                                        project.Zhuanjia = zhuanjia.TextContent.Trim();//OuterHtml
                                    }
                                    project.Creater = "111";
                                    project.Createtime = zhengfuProject.Createtime;
                                    project.Modifier = project.Creater;
                                    project.Modifytime = project.Createtime;
                                    project.RecStatus = "A";
                                    _ccontext1.Projects.Add(project);
                                    var table1 = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_noticeBidResult-noticeBidResult");
                                    if (table1 != null && table1.ToList().Count > 0)
                                    {
                                        var table = table1.First().QuerySelectorAll("table");
                                        if (table != null && table.ToList().Count > 0)
                                        {
                                            for (int i = 0; i < table.Length; i++)
                                            {
                                                var td = table[i].QuerySelectorAll("td");
                                                if (td.Length >= 3)
                                                {
                                                    var projectzhongbiao = new Projectzhongbiao();
                                                    projectzhongbiao.ZhongbiaoId = Guid.NewGuid().ToString();
                                                    projectzhongbiao.ProjectId = project.Id;
                                                    projectzhongbiao.Names = td[0].TextContent.Trim();
                                                    projectzhongbiao.Price = td[2].TextContent.Trim();
                                                    _ccontext1.Projectzhongbiaos.Add(projectzhongbiao);
                                                }
                                            }
                                        }
                                    }
                                string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    string result = string.Empty;
                                    Uri postUrl = new Uri(url);
                                    eswebcrawler eswebcrawler1 = new eswebcrawler();
@@ -576,7 +719,7 @@
                                    {
                                        throw new Exception("保存数据库出错!");
                                    }
                                    _ccontext1.SaveChanges();
                                    _ccontext.SaveChanges();
                                }
@@ -597,16 +740,16 @@
                }
                logg.WriteLog("四川省中标公告 获取完成");
                logg.WriteLog("四川省中标公告 获取完成", "四川");
                //翻页信息
                currPage = 1;
                pageSize = 10;
                //开始访问 采购公告
                logg.WriteLog("四川省更正公告 开始获取");
                logg.WriteLog("四川省更正公告 开始获取", "四川");
                while (true)
                {
                    logg.WriteLog(currPage.ToString());
                    logg.WriteLog(currPage.ToString(), "四川");
                    string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "&noticeType=00103&regionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
                    using (HttpClient client = new HttpClient())
                    {
@@ -781,6 +924,10 @@
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            break;
                                        default:
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            break;
                                    }
                                    zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
                                    zhengfuProject.Title = sichuanjieshoudtl.title;
@@ -803,7 +950,7 @@
                                    _ccontext.ZhengfuProjects.Add(zhengfuProject);
                                    string url = $"{api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    string result = string.Empty;
                                    Uri postUrl = new Uri(url);
                                    eswebcrawler eswebcrawler1 = new eswebcrawler();
@@ -877,17 +1024,17 @@
                }
                logg.WriteLog("四川省更正公告 获取完成");
                logg.WriteLog("四川省更正公告 获取完成", "四川");
                //翻页信息
                currPage = 1;
                pageSize = 10;
                //开始访问 采购公告
                logg.WriteLog("四川省废标公告 开始获取");
                logg.WriteLog("四川省废标公告 开始获取", "四川");
                while (true)
                {
                    logg.WriteLog(currPage.ToString());
                    logg.WriteLog(currPage.ToString(), "四川");
                    string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage=" + currPage + "&pageSize=" + pageSize + "&noticeType=001004,001006&regionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime=" + operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
                    using (HttpClient client = new HttpClient())
                    {
@@ -1062,6 +1209,10 @@
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            break;
                                        default:
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            break;
                                    }
                                    zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
                                    zhengfuProject.Title = sichuanjieshoudtl.title;
@@ -1084,7 +1235,63 @@
                                    _ccontext.ZhengfuProjects.Add(zhengfuProject);
                                    string url = $"{api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    var projects = _ccontext1.Projects.Where(x => x.RecStatus == "A" && x.Number == zhengfuProject.OpenTenderCode && x.Fblx != null ).ToList();
                                    if (projects.Count > 0)
                                    {
                                        foreach (var project in projects)
                                        {
                                            project.Fblx = "停止评审";
                                        }
                                        var parser = new HtmlParser();
                                        var document = parser.ParseDocument(zhengfuProject.Content);
                                        var contentList = document.All.Where(m => m.Id == "noticeArea");
                                        var cDocumet = contentList.First();
                                        var Numberss = cDocumet.QuerySelectorAll("p");
                                        foreach (var Number in Numberss)
                                        {
                                            if (Number != null && Number.TextContent.Contains("废标理由"))
                                            {
                                                var Fbyy = Number.QuerySelector("span");
                                                foreach(var project in projects)
                                                {
                                                    project.Fbyy = Fbyy.TextContent.Trim();
                                                    project.Fblx = "停止评审";
                                                }
                                            }
                                            if (Number != null && Number.TextContent.Contains("终止原因"))
                                            {
                                                var Fbyy = Number.QuerySelector("span");
                                                foreach (var project in projects)
                                                {
                                                    project.Fbyy = Fbyy.TextContent.Trim();
                                                    project.Fblx = "停止评审";
                                                }
                                            }
                                        }
                                        var feibiaoliyous = cDocumet.QuerySelectorAll("div").Where(m => m.Id == "_notice_content_reason-reason");
                                        if (feibiaoliyous != null && feibiaoliyous.ToList().Count > 0)
                                        {
                                            var feibiaoliyou = feibiaoliyous.First();
                                            foreach (var project in projects)
                                            {
                                                project.Fbyy = feibiaoliyou.TextContent.Replace("\n", "").Replace("\t", "");
                                                project.Fblx = "终止采购";
                                            }
                                        }
                                        _ccontext1.SaveChanges();
                                    }
                                    string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    string result = string.Empty;
                                    Uri postUrl = new Uri(url);
                                    eswebcrawler eswebcrawler1 = new eswebcrawler();
@@ -1158,16 +1365,16 @@
                }
                logg.WriteLog("四川省废标公告 获取完成");
                logg.WriteLog("四川省废标公告 获取完成", "四川");
                //翻页信息
                currPage = 1;
                 pageSize = 10;
                //开始访问 采购公告
                logg.WriteLog("四川省意向公开 开始获取");
                logg.WriteLog("四川省意向公开 开始获取", "四川");
                while (true)
                {
                    logg.WriteLog(currPage.ToString());
                    logg.WriteLog(currPage.ToString(), "四川");
                    string address = "https://zfcg.scsczt.cn/freecms/rest/v1/notice/selectInfoMoreChannel.do?&siteId=94c965cc-c55d-4f92-8469-d5875c68bd04&channel=c5bff13f-21ca-4dac-b158-cb40accd3035&currPage="+ currPage + "&pageSize="+ pageSize + "&noticeType=59&regionCode=&purchaseManner=&title=&openTenderCode=&purchaser=&agency=&purchaseNature=&operationStartTime="+ operationStartTime + "&operationEndTime=&selectTimeName=noticeTime&cityOrArea=";
                    using (HttpClient client = new HttpClient())
                    {
@@ -1342,6 +1549,10 @@
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            break;
                                        default:
                                            zhengfuProject.Cgfs = "10";
                                            cgfsName = "其他";
                                            break;
                                    }
                                    zhengfuProject.OpenTenderCode = sichuanjieshoudtl.openTenderCode;
                                    zhengfuProject.Title = sichuanjieshoudtl.title;
@@ -1364,7 +1575,7 @@
                                    _ccontext.ZhengfuProjects.Add(zhengfuProject);
                                    string url = $"{api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    string url = $"{Program.api_domain}/webcrawler/_doc/" + zhengfuProject.Id;
                                    string result = string.Empty;
                                    Uri postUrl = new Uri(url);
                                    eswebcrawler eswebcrawler1 = new eswebcrawler();
@@ -1438,7 +1649,7 @@
                }
                logg.WriteLog("四川省意向公开 获取完成");
                logg.WriteLog("四川省意向公开 获取完成", "四川");
                //保存最后更新时间
                if (updatetime != null)
@@ -1458,11 +1669,11 @@
            }
            catch (Exception ex)
            {
                logg.WriteLog(ex.Message);
                logg.WriteLog(ex.Message, "四川");
                logg.WriteLog(ex.StackTrace);
                logg.WriteLog(ex.StackTrace, "四川");
                logg.WriteLog(ex.ToString());
                logg.WriteLog(ex.ToString(), "四川");
            }
            finally
            {