static void SyncDataOne(string url, string table, int page)
{
Console.WriteLine();
//下载文件
byte[] b = new System.Net.WebClient().DownloadData(url);
//json接口返回
string json = System.Text.Encoding.UTF8.GetString(b);
//计算hash
string md5 = utilty.Md5(json);
//获取数据集
System.Data.DataTable dataTable = utilty.JsonToDataTable(url, json, new string[] { "data", "result" });
//string md5 = dataTable.Rows.Count.ToString();
Console.Write("文件{0}已下载，包含数据{1}行。",url,dataTable.Rows.Count);
if (dataTable.Rows.Count == 0)
{
Console.Write("任务结束！");
return;
}
if (dataTable.Rows.Count < 1000)
{
Console.Write("正在检查数据量...");
using (var conn = new SqlConnection())
{
conn.ConnectionString = ConfigurationManager.AppSettings["conn"];
conn.Open();
//检查数据库md5
using (SqlCommand command = conn.CreateCommand())
{
command.CommandText = "select md5 from tb_download where url = @url and status=1";
//by于
//Console.Write(command.CommandText);
command.Parameters.AddWithValue("url", url);
object obj = command.ExecuteScalar();
//如果存在且md5相同则跳过处理。
if (obj != null && obj.ToString() == md5)
{
Console.Write("无更新数据。");
return;
}
//更新插入
command.CommandText = @"delete from tb_download where url = @url;
insert into tb_download(id,url,data,md5,rowtime,status)
values(newid(),@url,@data,@md5,getdate(),0)";
command.Parameters.AddWithValue("md5", md5);
command.Parameters.AddWithValue("data", json);
command.ExecuteNonQuery();
//删除更新数据
command.CommandText = string.Format("delete from {0} where url=@url", table);
command.ExecuteNonQuery();
//有变化的根据url更新目标表
using (System.Data.SqlClient.SqlBulkCopy bcp = new SqlBulkCopy(conn))
{
foreach (System.Data.DataColumn c in dataTable.Columns)
{
bcp.ColumnMappings.Add(c.ColumnName, c.ColumnName);
}
bcp.DestinationTableName = table;
try
{
bcp.WriteToServer(dataTable);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
//更新任务状态
command.CommandText = "update tb_download set status=1 where url = @url";
command.ExecuteNonQuery();
}
//by 20200529
if (dataTable.Rows.Count == 1000)
{
Console.WriteLine("第二页");
page = 2;
if (page == 2)
{
url = url + "&page=" + page;
SyncDataOne(url, table, 2);
}
}
if (page == 2)
{
Console.Write("第二页任务结束！");
return;
}
//by end
//更新任务状态
//command.CommandText = "exec sync_done";
//command.ExecuteNonQuery();
}
Console.Write("记录已更新。");
}
改了标红的代码 就报错了.. 条数等于1000时不执行..
