using AngleSharp.Html.Parser;
|
using Crawler.sichuan;
|
using Microsoft.EntityFrameworkCore;
|
using Microsoft.Extensions.DependencyInjection;
|
using Newtonsoft.Json;
|
using OpenQA.Selenium;
|
using OpenQA.Selenium.Chrome;
|
using System;
|
using System.Linq;
|
using System.Net;
|
using System.Net.Http;
|
using System.Net.Http.Headers;
|
using System.Threading;
|
using zhengcaioa.Models;
|
|
namespace Crawler
|
{
|
class Program
|
{
|
|
//public static string connection = @"server=.;database=WebCrawler;uid=sa;pwd=123456;";
|
//public static string connection1 = @"server=.;database=zhengcaioa;uid=sa;pwd=123456;";
|
//public static string api_domain = "http://192.168.0.9:9200";
|
|
public static string connection = @"server=172.26.97.184;database=WebCrawler;uid=sa;pwd=Za20222812;";
|
public static string connection1 = @"server=172.26.97.184;database=zhengcaioa;uid=sa;pwd=Za20222812;";
|
public static string api_domain = "http://172.26.97.184:9200";
|
static void Main(string[] args)
|
{
|
|
|
|
|
|
|
////FiBook fiBook = new FiBook();
|
////fiBook.Id = Guid.NewGuid().ToString();
|
////fiBook.RecStatus = "A";
|
////fiBook.Creater = "sdfdsfdsf";
|
////fiBook.Createtime = DateTime.Now;
|
////fiBook.Modifier = "sdfdsfdsf";
|
////fiBook.Modifytime = DateTime.Now;
|
////cccontext.FiBooks.Add(fiBook);
|
////cccontext.SaveChanges();
|
|
//ZhengfuProject zhengfuProject = new ZhengfuProject();
|
|
//logg.WriteLog("测试写日志");
|
|
//ChromeOptions options = new ChromeOptions();
|
//options.BinaryLocation = "C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\Application\\chrome.exe";
|
////options.AddArgument("--headless");
|
////options.AddArgument("blink-settings=imagesEnabled=false");
|
////_logger.LogInformation(address + " 开始解析");
|
////string chromeDriverDirectory = Microsoft.DotNet.PlatformAbstractions.ApplicationEnvironment.ApplicationBasePath;
|
////_logger.LogInformation(chromeDriverDirectory + " 当前项目路径");
|
//try
|
//{
|
// var driver = new ChromeDriver("C:\\Users\\Administrator\\AppData\\Local\\Google\\Chrome\\Application\\", options);
|
// driver.Navigate().GoToUrl("http://www.mof.gov.cn/gkml/xinxi/zhongyangbiaoxun/zhongyangzhaobiaogonggao/index_21.htm");
|
// //_logger.LogInformation(address + " 接着开始");
|
// while (true)
|
// {
|
// var content = driver.FindElement(By.ClassName("xwbd_lianbolistfrcon")).FindElements(By.TagName("li"));//FindElements(By.XPath("/html/body/section/div/div/div[2]/div/ul/li"));
|
|
|
// foreach (var item in content)
|
// {
|
// var link = item.FindElement(By.XPath("child::a")).GetAttribute("href");
|
// var title = item.Text;
|
// //if (!dic.ContainsKey(link))
|
// // dic.Add(link, title);
|
// }
|
// //_logger.LogInformation(" 开始翻页");
|
// //var nextPage = driver.FindElements(By.XPath("//*[@id='pagination']/ul/li"));
|
// var nextPage = driver.FindElement(By.ClassName("pagerji")).FindElements(By.TagName("span"));
|
// var boolll = false;
|
// foreach (var nextPagea in nextPage)
|
// {
|
// var linkk = nextPagea.FindElement(By.XPath("child::a")).GetAttribute("href");
|
// if (">".Equals(nextPagea.Text))
|
// {
|
// if (linkk.IndexOf("#") < 0)
|
// {
|
// nextPagea.Click();
|
|
|
// System.Threading.Thread.Sleep(1000); // 延时，避免找不到元素
|
// }
|
// else
|
// {
|
// boolll = true;
|
// }
|
// break;
|
// }
|
|
// //var sss1 = nextPagea.ToString();
|
// //var sss2 = nextPagea.Text;
|
// //var sss = nextPagea.GetAttribute("class");
|
// //if (">".Equals(nextPagea.Text))
|
// //{
|
// // if (!"disabled".Equals(nextPagea.GetAttribute("class")))
|
// // {
|
// // nextPagea.Click();
|
|
|
// // System.Threading.Thread.Sleep(1000); // 延时，避免找不到元素
|
// // }
|
// // else
|
// // {
|
// // boolll = true;
|
// // }
|
// // break;
|
// //}
|
// }
|
// if (boolll)
|
// {
|
// break;
|
// }
|
|
// }
|
|
// driver.Quit();
|
//}
|
//catch (Exception ex)
|
//{
|
// //_logger.LogInformation(ex.Message);
|
|
// //_logger.LogInformation(ex.StackTrace);
|
|
// //_logger.LogInformation(ex.ToString());
|
//}
|
//finally
|
//{
|
|
//}
|
|
|
|
|
Program obj = new Program();
|
//方法一：调用线程执行方法，在方法中实现死循环，每个循环Sleep设定时间
|
Thread thread = new Thread(new ThreadStart(obj.Method1));
|
thread.Start();
|
|
|
Thread thread1 = new Thread(new ThreadStart(obj.Method2));
|
thread1.Start();
|
|
|
Thread thread2 = new Thread(new ThreadStart(obj.Method3));
|
thread2.Start();
|
|
Console.WriteLine("Hello World!");
|
}
|
|
void Method1()
|
{
|
while (true)
|
{
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
var dbContextOptionBuilder = new DbContextOptionsBuilder<WebCrawlerContext>();
|
var cccontext = new WebCrawlerContext(dbContextOptionBuilder.UseSqlServer(connection).Options);
|
|
var dbContextOptionBuilder1 = new DbContextOptionsBuilder<zhengcaioaContext>();
|
var cccontext1 = new zhengcaioaContext(dbContextOptionBuilder1.UseSqlServer(connection1).Options);
|
|
//获取四川的政采数据
|
sichuanoperation.operations(cccontext, cccontext1);
|
//Console.WriteLine("这个暂停");
|
Thread.CurrentThread.Join(1000 * 60 * 60 * 2);//阻止设定时间
|
}
|
}
|
|
void Method2()
|
{
|
while (true)
|
{
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
var dbContextOptionBuilder = new DbContextOptionsBuilder<WebCrawlerContext>();
|
var cccontext = new WebCrawlerContext(dbContextOptionBuilder.UseSqlServer(connection).Options);
|
|
var dbContextOptionBuilder1 = new DbContextOptionsBuilder<zhengcaioaContext>();
|
var cccontext1 = new zhengcaioaContext(dbContextOptionBuilder1.UseSqlServer(connection1).Options);
|
|
//获取四川的政采数据
|
OldSichuanoperation.operations(cccontext, cccontext1);
|
Thread.CurrentThread.Join(1000 * 60 * 60 * 2);//阻止设定时间
|
}
|
}
|
|
|
void Method3()
|
{
|
while (true)
|
{
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
var dbContextOptionBuilder = new DbContextOptionsBuilder<WebCrawlerContext>();
|
var cccontext = new WebCrawlerContext(dbContextOptionBuilder.UseSqlServer(connection).Options);
|
|
//var dbContextOptionBuilder1 = new DbContextOptionsBuilder<zhengcaioaContext>();
|
//var cccontext1 = new zhengcaioaContext(dbContextOptionBuilder1.UseSqlServer(connection1).Options);
|
|
//获取中国政采网的四川政采数据
|
CcgpSichuanoperation.operations(cccontext);
|
Thread.CurrentThread.Join(1000 * 60 * 10);//阻止设定时间
|
}
|
}
|
|
|
|
|
|
|
|
|
}
|
}
|
