SQL 程序集 WebResponse 和字符串解析非常慢
SQL Assembly WebResponse and String Parsing VERY slow
所以我正在快速学习C#的方法(继承这个问题的完全菜鸟);我已经编写了以下代码,它调用一个 returns JSON 并不总是格式正确的 Web 服务。这里的任务是获取 JSON 字符串并将其分解为数组段,这些段被插入到 SQL table 中以供进一步解析和测试。 IE。如果 return 字符串类似于
{1234:{5678:{1:{"key":"val","key":"val"},{2:{"key":"val","key":"val"}}}}
那么行将是:
{1234}
{5678}
{1:{"key":"val","key":"val"}
{2:{"key":"val","key":"val"}
这是 .NET 3.0 和 SQL Server 2008 R2(旧版)。
这是我的工作代码:
public partial class UserDefinedFunctions
{
[Microsoft.SqlServer.Server.SqlFunction(DataAccess =
DataAccessKind.Read)]
public static SqlString TestParse(SqlString uri, SqlString username, SqlString passwd, SqlString postdata)
{
//-----
// The SqlPipe is how we send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
SqlString document;
try
{
// Set up the request, including authentication
WebRequest req = WebRequest.Create(Convert.ToString(uri));
if (Convert.ToString(username) != null & Convert.ToString(username) != "")
{
req.Credentials = new NetworkCredential(
Convert.ToString(username),
Convert.ToString(passwd));
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
//SqlContext.Pipe.Send("...get the data");
using (StreamReader rdr = new StreamReader(dataStream))
{
document = (SqlString)rdr.ReadToEnd();
rdr.Close();
//-----
string connectionString = null;
string sql = null;
connectionString = "Data source= 192.168.0.5; Database=Administration;User Id=Foo;Password=Blah; Trusted_Connection=True;";
using (SqlConnection cnn = new SqlConnection(connectionString))
{
sql = "INSERT INTO JSON_DATA (JSONROW) VALUES(@data)";
cnn.Open();
using (SqlCommand cmd = new SqlCommand(sql, cnn))
{
String payload = "";
String nestpayload = "";
int nests = 0;
String json = document.ToString();
/*first lets do some housekeeping on our payload; double closing curly braces need to be escaped (with curly braces!) in order to keep them in the string.*/
json = json.Replace("\", "");
int i = json.Length;
//return new SqlString(json);
while (i > 1)
{
/*find the first closing "}" in the string and then check to see if there are more than one.
We need to read the data up to each closing brace, pull off that substring and process it for each iteration until the string is gone.*/
int closingbrace = json.IndexOf("}"); //First closing brace
int nextbrace = Math.Max(0, json.IndexOf("{", closingbrace)); //Next opening brace
String ChkVal = json.Substring(closingbrace + 1, Math.Max(1, nextbrace - closingbrace)); //+1 to ignore the 1st closing brace
int checks = Math.Max(0, ChkVal.Length) - Math.Max(0, ChkVal.Replace("}", "").Length);
payload = json.Substring(0, Math.Max(0, (json.IndexOf("}") + 1)));
/*Remove the payload from the string*/
json = json.Substring(payload.Length + 1);
/*"nests" is how many nested levels excluding the opening brace for the closing brace we found.*/
nests = (payload.Length - payload.Replace("{", "").Length);
/*If we have more then one nest level check to see if any of them go with the payload*/
if (nests > 1)
{
/*Break out the nested section and remove it from the payload.*/
nestpayload = payload.Substring(0, payload.LastIndexOf("{"));
payload = payload.Substring(payload.LastIndexOf("{"), payload.Length - payload.LastIndexOf("{"));
while (nests > 1)
{
if (checks > 0) //# of right braces in payload equals number of left-side nests go with the payload
{
// payload = nestpayload.Substring(Math.Max(0, nestpayload.LastIndexOf("{")), Math.Max(0, nestpayload.Length) - Math.Max(0, (nestpayload.LastIndexOf("{")))) + payload;//The second Math.Max defaults to 1; if we got here there is at minimum one "{" character in the substring
payload = nestpayload.Substring(nestpayload.LastIndexOf("{")) + payload;
nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0, nestpayload.LastIndexOf("{"))));
checks--;
nests--;
}
else
{
/*If we got here there are no more pieces of the nested data to append to the payload.
We use an array and string.split to keep the nest ordering correct.*/
string[] OrderedNest = nestpayload.Split('{');
for (int s = 0; s < OrderedNest.Length; s++)
{
if (OrderedNest[s] != "")
{
cmd.Parameters.AddWithValue("@data", "{" + OrderedNest[s].Replace(":", "}"));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
}
//cmd.Parameters.AddWithValue("@data", nestpayload.Substring(Math.Max(0,nestpayload.LastIndexOf("{"))).Replace(":","}"));
//cmd.Parameters.AddWithValue("@data", OrderedNest[1].Replace(":","}")+OrderedNest[2]);
// cmd.ExecuteNonQuery();
//cmd.Parameters.Clear();
//nests = Math.Max(0, nests - 1);
nests = 0;
//nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0,nestpayload.LastIndexOf("{"))));
}
}
}
/*At the very end payload will be a single "}"; check for this and discard the last row*/
if (payload != "}")
{
cmd.Parameters.AddWithValue("@data", new SqlChars(payload));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
/*Get the new string length*/
i = json.Length;
payload = "";
}
}
}
//-----
/* }
catch (Exception e)
{
return e.ToString();
}*/
}
// Close up everything...
dataStream.Close();
}
resp.Close();
// .. and return the output to the caller.
}//end using
return ("Finished");
}
catch (WebException e)
{
throw e;
}
}
}
虽然它可以工作,但速度非常慢;将 1500 行写入服务器需要 4 分钟以上。每天一次,这将需要写入约 60,000 条记录;其余时间可能会有 100 条记录 POSTED 和 returned(我还没有完成 POST 部分)。我敢肯定有很多我在这里做的事情不太合适导致了问题,但我完全不知道从哪里开始。我很兴奋,我可以从中得到正确的回应!任何 ideas/thoughts/help/sympathy 将不胜感激。
这里有几个问题,其中最重要的是您似乎已经在 public 互联网上发布了您的“sa”密码。以下是我看到的代码问题:
- 虽然可以在 SQLCLR 中执行 Web 服务调用,但这绝对是一个高级主题,充满了陷阱。这不是 SQLCLR 新手/初学者应该做的事情,CLR 本身已经是常规 .NET 编程的细微差别。
- 删除
SqlPipe
行及其上方的注释行。函数不会通过 SqlPipe
将数据传回给调用者;用于存储过程。
- 您可能不应该使用
WebRequest
document
应该是 string
,而不是 SqlString
。您永远不会 return document
,只会将其转换回 string
,所以应该就是这样。
- 使用
HttpWebRequest
而不是 WebRequest
。这样您就不必偶尔将其转换为 HttpWebRequest
.
- 不要将
SqlString
输入参数转换为 string
(例如 Convert.ToString(uri)
)。所有 Sql*
类型都有一个 Value
属性 return 是本机 .NET 类型中的值。因此,只需使用 uri.Value
,依此类推。
- 不要通过
Convert.ToString(username) != null
检查 NULL
输入。所有 Sql*
类型都有一个 IsNull
属性 您可以检查。所以改为使用 !username.IsNull
.
- 不要在保持远程
HttpWebRequest
连接打开的同时进行所有文本处理(尤其是与另一个系统联系以进行逐行插入的处理)。 only 你应该在 using (WebResponse resp = req.GetResponse())
中做的事情是填充 document
变量。不要对 document
的内容进行任何处理,直到您离开最外层的 using()
.
- 不要单独插入(即
while (i > 1)
循环)。他们甚至不在交易中。如果您在文档中间出现错误,您将加载部分数据(除非此过程没问题)。
- ALWAYS 模式限定数据库对象。意思是,
JSON_DATA
应该是 dbo.JSON_DATA
(或者如果不是 dbo
,则使用任何模式)。
- 在您的
connectionString
中,您同时拥有 Id/Password 和 Trusted_Connection
。不要同时使用两者,因为它们是相互排斥的选项(如果两者都有,则 Id/Password 将被忽略,仅使用 Trusted_Connection
)。
- 请不要以
sa
登录或让您的应用程序以 sa
登录。那简直是自讨苦吃
- 您连接的 SQL 服务器实例是否与此 SQLCLR 对象 运行 上的实例不同?如果它是同一个实例,您最好将其更改为
SqlProcedure
以便您可以使用 Context_Connection=True;
作为连接字符串。这是附加到从中调用它的会话的进程内连接。
- 不要使用
Parameters.AddWithValue()
。馊主意。使用特定且适当的数据类型创建 SqlParameter。然后通过 Add()
. 添加到 Parameters
集合
可能还有其他问题,但这些是显而易见的问题。正如我在第 1 点中所说的那样,您可能在这里不知所措。不是试图消极,只是试图避免 SQLCLR 的另一个糟糕实现,这通常会导致对这个非常有用的功能的负面看法。如果你想追求这个,那么请先对 SQLCLR 的工作原理、最佳实践等进行更多研究。一个很好的起点是我在 SQL 服务器上写的关于这个主题的系列文章中央:Stairway to SQLCLR.
或者,另一种选择是使用 INET_GetWebPages SQLCLR TVF,它在 SQL# SQLCLR 库(我写的)。这个选项不是免费的,但它可以让你简单地安装网络请求部分,然后你只需要在 SQLCLR 标量 UDF 中单独解析 returned 文档(这可能是最好的无论如何,即使您自己执行网络请求功能/存储过程)。事实上,如果您在 SQL 服务器的同一实例中插入 table,您可以为文档解析器创建一个 SQLCLR TVF 并传递每个 OrderedNest
值使用 yield return
返回(将结果流回)并按如下方式使用:
DECLARE @JSON NVARCHAR(MAX);
SELECT @JSON = [content]
FROM SQL#.INET_GetWebPages(@uri, .....);
INSERT INTO dbo.JSON_DATA (JSONROW)
SELECT [column_name]
FROM dbo.MyBrokenJsonFixerUpper(@JSON);
祝你好运!
我将此问题标记为已回答,因为很明显,需要的是重写和重新思考我的原始脚本。 @Solomon Rutzky 赞成提供有用的信息,这些信息使我得出了这个结论。对于那些
这里感兴趣的是重写:
using System;
using System.Data;
using System.Data.SqlClient;
using System.Data.SqlTypes;
using Microsoft.SqlServer.Server;
using System.Collections;
using System.Globalization;
// Other things we need for WebRequest
using System.Net;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
public partial class StoredProcedures
{
[Microsoft.SqlServer.Server.SqlProcedure]
public static void ApiParser(SqlString uri, SqlString user, SqlString pwd, SqlString postd)
{
// Create an SqlPipe to send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
//Make sure we have a url to process
if (uri.IsNull || uri.Value.Trim() == string.Empty)
{
pipe.Send("uri cannot be empty");
return;
}
try
{
//Create our datatable and get the table structure from the database
DataTable table = new DataTable();
string connectionString = null;
//connectionString = "Data source= 192.168.0.5; Database=Administration; Trusted_Connection=True;";
connectionString = "Data Source=(localdb)\ProjectsV12;Integrated Security=True;Connect Timeout=30;Encrypt=False;TrustServerCertificate=False;ApplicationIntent=ReadWrite;MultiSubnetFailover=False";
using (SqlConnection gts = new SqlConnection(connectionString))
{
gts.Open();
using (SqlDataAdapter adapter = new SqlDataAdapter("SELECT TOP 0 * FROM sp_WebSvcs.dbo.JSON_DATA", gts))
{
adapter.Fill(table);
}
}
// Send a message string back to the client.
pipe.Send("Beginning Api Call...");
String json = "";
// Set up the request, including authentication
WebRequest req = HttpWebRequest.Create(uri.Value);
if (!user.IsNull & user.Value != "")
{
req.Credentials = new NetworkCredential(user.Value, pwd.Value);
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
using (StreamReader rdr = new StreamReader(dataStream))
{
json = (String)rdr.ReadToEnd();
rdr.Close();
}
// Close up everything...
dataStream.Close();
}
resp.Close();
}//end using resp
pipe.Send("Api Call complete; Parsing returned data...");
int i = 0;
String h = "";
String l = "";
int s = 0;
int p = 0;
int b = 0;
int payload = 0;
foreach (string line in json.Split(new[] { "}," }, StringSplitOptions.None))
{
if (line != "")
{
l = line;
i = l.Replace("{", "").Length + 1;
p = l.LastIndexOf("{");
if (line.Length > i) //we find this at the beginning of a group of arrays
{
h = line.Substring(0, p - 1);
s = Math.Max(0, h.LastIndexOf("{"));
if (h.Length > s && s != 0)
/*We have a nested array that has more than one level.
*This should only occur at the beginning of new array group.
*Advance the payload counter and get the correct string from line.*/
{
payload++;
l = line.Substring(s, line.Length - s);
}
h = (s >= 0) ? h.Substring(0, s) : h;
//=============
/*At this point 'h' is a nest collection. Split and add to table.*/
string[] OrderedNest = h.Split('{');
for (int z = 0; z < OrderedNest.Length; z++)
{
if (OrderedNest[z] != "")
{
table.Rows.Add(payload, "{" + OrderedNest[z].Replace(":", "").Replace("[","").Replace("]","") + "}");
}
}
//=============
}
else
{
h = null;
}
//at this point the first character in the row should be a "{"; If not we need to add one.
if (l[0].ToString() != "{")
{
l = "{" + l;
}
if (l.Replace("{", "").Length != l.Replace("}", "").Length) //opening and closing braces don't match; match the closing to the opening
{
l = l.Replace("}", "");
b = l.Length - l.Replace("{", "").Length;
l = l + new String('}', b);
}
table.Rows.Add(payload, l.Replace("\\"", "").Replace("\", "").Replace("]","").Replace("[",""));
}
}
//====
using (SqlConnection cnn = new SqlConnection(connectionString))
{
cnn.Open();
using (SqlBulkCopy copy = new SqlBulkCopy(cnn))
{
copy.DestinationTableName = "sp_WebSvcs.dbo.JSON_DATA";
copy.WriteToServer(table);
}
}
//====
} //end try
catch (Exception e)
{
pipe.Send("We have a problem!");
throw new Exception("\n\n" + e.Message + "\n\n");
}
pipe.Send("Parsing complete");
}
}
所以我正在快速学习C#的方法(继承这个问题的完全菜鸟);我已经编写了以下代码,它调用一个 returns JSON 并不总是格式正确的 Web 服务。这里的任务是获取 JSON 字符串并将其分解为数组段,这些段被插入到 SQL table 中以供进一步解析和测试。 IE。如果 return 字符串类似于
{1234:{5678:{1:{"key":"val","key":"val"},{2:{"key":"val","key":"val"}}}}
那么行将是:
{1234}
{5678}
{1:{"key":"val","key":"val"}
{2:{"key":"val","key":"val"}
这是 .NET 3.0 和 SQL Server 2008 R2(旧版)。 这是我的工作代码:
public partial class UserDefinedFunctions
{
[Microsoft.SqlServer.Server.SqlFunction(DataAccess =
DataAccessKind.Read)]
public static SqlString TestParse(SqlString uri, SqlString username, SqlString passwd, SqlString postdata)
{
//-----
// The SqlPipe is how we send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
SqlString document;
try
{
// Set up the request, including authentication
WebRequest req = WebRequest.Create(Convert.ToString(uri));
if (Convert.ToString(username) != null & Convert.ToString(username) != "")
{
req.Credentials = new NetworkCredential(
Convert.ToString(username),
Convert.ToString(passwd));
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
//SqlContext.Pipe.Send("...get the data");
using (StreamReader rdr = new StreamReader(dataStream))
{
document = (SqlString)rdr.ReadToEnd();
rdr.Close();
//-----
string connectionString = null;
string sql = null;
connectionString = "Data source= 192.168.0.5; Database=Administration;User Id=Foo;Password=Blah; Trusted_Connection=True;";
using (SqlConnection cnn = new SqlConnection(connectionString))
{
sql = "INSERT INTO JSON_DATA (JSONROW) VALUES(@data)";
cnn.Open();
using (SqlCommand cmd = new SqlCommand(sql, cnn))
{
String payload = "";
String nestpayload = "";
int nests = 0;
String json = document.ToString();
/*first lets do some housekeeping on our payload; double closing curly braces need to be escaped (with curly braces!) in order to keep them in the string.*/
json = json.Replace("\", "");
int i = json.Length;
//return new SqlString(json);
while (i > 1)
{
/*find the first closing "}" in the string and then check to see if there are more than one.
We need to read the data up to each closing brace, pull off that substring and process it for each iteration until the string is gone.*/
int closingbrace = json.IndexOf("}"); //First closing brace
int nextbrace = Math.Max(0, json.IndexOf("{", closingbrace)); //Next opening brace
String ChkVal = json.Substring(closingbrace + 1, Math.Max(1, nextbrace - closingbrace)); //+1 to ignore the 1st closing brace
int checks = Math.Max(0, ChkVal.Length) - Math.Max(0, ChkVal.Replace("}", "").Length);
payload = json.Substring(0, Math.Max(0, (json.IndexOf("}") + 1)));
/*Remove the payload from the string*/
json = json.Substring(payload.Length + 1);
/*"nests" is how many nested levels excluding the opening brace for the closing brace we found.*/
nests = (payload.Length - payload.Replace("{", "").Length);
/*If we have more then one nest level check to see if any of them go with the payload*/
if (nests > 1)
{
/*Break out the nested section and remove it from the payload.*/
nestpayload = payload.Substring(0, payload.LastIndexOf("{"));
payload = payload.Substring(payload.LastIndexOf("{"), payload.Length - payload.LastIndexOf("{"));
while (nests > 1)
{
if (checks > 0) //# of right braces in payload equals number of left-side nests go with the payload
{
// payload = nestpayload.Substring(Math.Max(0, nestpayload.LastIndexOf("{")), Math.Max(0, nestpayload.Length) - Math.Max(0, (nestpayload.LastIndexOf("{")))) + payload;//The second Math.Max defaults to 1; if we got here there is at minimum one "{" character in the substring
payload = nestpayload.Substring(nestpayload.LastIndexOf("{")) + payload;
nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0, nestpayload.LastIndexOf("{"))));
checks--;
nests--;
}
else
{
/*If we got here there are no more pieces of the nested data to append to the payload.
We use an array and string.split to keep the nest ordering correct.*/
string[] OrderedNest = nestpayload.Split('{');
for (int s = 0; s < OrderedNest.Length; s++)
{
if (OrderedNest[s] != "")
{
cmd.Parameters.AddWithValue("@data", "{" + OrderedNest[s].Replace(":", "}"));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
}
//cmd.Parameters.AddWithValue("@data", nestpayload.Substring(Math.Max(0,nestpayload.LastIndexOf("{"))).Replace(":","}"));
//cmd.Parameters.AddWithValue("@data", OrderedNest[1].Replace(":","}")+OrderedNest[2]);
// cmd.ExecuteNonQuery();
//cmd.Parameters.Clear();
//nests = Math.Max(0, nests - 1);
nests = 0;
//nestpayload = nestpayload.Substring(0, Math.Max(0, Math.Max(0,nestpayload.LastIndexOf("{"))));
}
}
}
/*At the very end payload will be a single "}"; check for this and discard the last row*/
if (payload != "}")
{
cmd.Parameters.AddWithValue("@data", new SqlChars(payload));
cmd.ExecuteNonQuery();
cmd.Parameters.Clear();
}
/*Get the new string length*/
i = json.Length;
payload = "";
}
}
}
//-----
/* }
catch (Exception e)
{
return e.ToString();
}*/
}
// Close up everything...
dataStream.Close();
}
resp.Close();
// .. and return the output to the caller.
}//end using
return ("Finished");
}
catch (WebException e)
{
throw e;
}
}
}
虽然它可以工作,但速度非常慢;将 1500 行写入服务器需要 4 分钟以上。每天一次,这将需要写入约 60,000 条记录;其余时间可能会有 100 条记录 POSTED 和 returned(我还没有完成 POST 部分)。我敢肯定有很多我在这里做的事情不太合适导致了问题,但我完全不知道从哪里开始。我很兴奋,我可以从中得到正确的回应!任何 ideas/thoughts/help/sympathy 将不胜感激。
这里有几个问题,其中最重要的是您似乎已经在 public 互联网上发布了您的“sa”密码。以下是我看到的代码问题:
- 虽然可以在 SQLCLR 中执行 Web 服务调用,但这绝对是一个高级主题,充满了陷阱。这不是 SQLCLR 新手/初学者应该做的事情,CLR 本身已经是常规 .NET 编程的细微差别。
- 删除
SqlPipe
行及其上方的注释行。函数不会通过SqlPipe
将数据传回给调用者;用于存储过程。 - 您可能不应该使用
WebRequest
document
应该是string
,而不是SqlString
。您永远不会 returndocument
,只会将其转换回string
,所以应该就是这样。- 使用
HttpWebRequest
而不是WebRequest
。这样您就不必偶尔将其转换为HttpWebRequest
. - 不要将
SqlString
输入参数转换为string
(例如Convert.ToString(uri)
)。所有Sql*
类型都有一个Value
属性 return 是本机 .NET 类型中的值。因此,只需使用uri.Value
,依此类推。 - 不要通过
Convert.ToString(username) != null
检查NULL
输入。所有Sql*
类型都有一个IsNull
属性 您可以检查。所以改为使用!username.IsNull
. - 不要在保持远程
HttpWebRequest
连接打开的同时进行所有文本处理(尤其是与另一个系统联系以进行逐行插入的处理)。 only 你应该在using (WebResponse resp = req.GetResponse())
中做的事情是填充document
变量。不要对document
的内容进行任何处理,直到您离开最外层的using()
. - 不要单独插入(即
while (i > 1)
循环)。他们甚至不在交易中。如果您在文档中间出现错误,您将加载部分数据(除非此过程没问题)。 - ALWAYS 模式限定数据库对象。意思是,
JSON_DATA
应该是dbo.JSON_DATA
(或者如果不是dbo
,则使用任何模式)。 - 在您的
connectionString
中,您同时拥有 Id/Password 和Trusted_Connection
。不要同时使用两者,因为它们是相互排斥的选项(如果两者都有,则 Id/Password 将被忽略,仅使用Trusted_Connection
)。 - 请不要以
sa
登录或让您的应用程序以sa
登录。那简直是自讨苦吃 - 您连接的 SQL 服务器实例是否与此 SQLCLR 对象 运行 上的实例不同?如果它是同一个实例,您最好将其更改为
SqlProcedure
以便您可以使用Context_Connection=True;
作为连接字符串。这是附加到从中调用它的会话的进程内连接。 - 不要使用
Parameters.AddWithValue()
。馊主意。使用特定且适当的数据类型创建 SqlParameter。然后通过Add()
. 添加到
Parameters
集合
可能还有其他问题,但这些是显而易见的问题。正如我在第 1 点中所说的那样,您可能在这里不知所措。不是试图消极,只是试图避免 SQLCLR 的另一个糟糕实现,这通常会导致对这个非常有用的功能的负面看法。如果你想追求这个,那么请先对 SQLCLR 的工作原理、最佳实践等进行更多研究。一个很好的起点是我在 SQL 服务器上写的关于这个主题的系列文章中央:Stairway to SQLCLR.
或者,另一种选择是使用 INET_GetWebPages SQLCLR TVF,它在 SQL# SQLCLR 库(我写的)。这个选项不是免费的,但它可以让你简单地安装网络请求部分,然后你只需要在 SQLCLR 标量 UDF 中单独解析 returned 文档(这可能是最好的无论如何,即使您自己执行网络请求功能/存储过程)。事实上,如果您在 SQL 服务器的同一实例中插入 table,您可以为文档解析器创建一个 SQLCLR TVF 并传递每个 OrderedNest
值使用 yield return
返回(将结果流回)并按如下方式使用:
DECLARE @JSON NVARCHAR(MAX);
SELECT @JSON = [content]
FROM SQL#.INET_GetWebPages(@uri, .....);
INSERT INTO dbo.JSON_DATA (JSONROW)
SELECT [column_name]
FROM dbo.MyBrokenJsonFixerUpper(@JSON);
祝你好运!
我将此问题标记为已回答,因为很明显,需要的是重写和重新思考我的原始脚本。 @Solomon Rutzky 赞成提供有用的信息,这些信息使我得出了这个结论。对于那些 这里感兴趣的是重写:
using System;
using System.Data;
using System.Data.SqlClient;
using System.Data.SqlTypes;
using Microsoft.SqlServer.Server;
using System.Collections;
using System.Globalization;
// Other things we need for WebRequest
using System.Net;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
public partial class StoredProcedures
{
[Microsoft.SqlServer.Server.SqlProcedure]
public static void ApiParser(SqlString uri, SqlString user, SqlString pwd, SqlString postd)
{
// Create an SqlPipe to send data back to the caller
SqlPipe pipe = SqlContext.Pipe;
//Make sure we have a url to process
if (uri.IsNull || uri.Value.Trim() == string.Empty)
{
pipe.Send("uri cannot be empty");
return;
}
try
{
//Create our datatable and get the table structure from the database
DataTable table = new DataTable();
string connectionString = null;
//connectionString = "Data source= 192.168.0.5; Database=Administration; Trusted_Connection=True;";
connectionString = "Data Source=(localdb)\ProjectsV12;Integrated Security=True;Connect Timeout=30;Encrypt=False;TrustServerCertificate=False;ApplicationIntent=ReadWrite;MultiSubnetFailover=False";
using (SqlConnection gts = new SqlConnection(connectionString))
{
gts.Open();
using (SqlDataAdapter adapter = new SqlDataAdapter("SELECT TOP 0 * FROM sp_WebSvcs.dbo.JSON_DATA", gts))
{
adapter.Fill(table);
}
}
// Send a message string back to the client.
pipe.Send("Beginning Api Call...");
String json = "";
// Set up the request, including authentication
WebRequest req = HttpWebRequest.Create(uri.Value);
if (!user.IsNull & user.Value != "")
{
req.Credentials = new NetworkCredential(user.Value, pwd.Value);
}
((HttpWebRequest)req).UserAgent = "CLR web client on SQL Server";
// Fire off the request and retrieve the response.
using (WebResponse resp = req.GetResponse())
{
using (Stream dataStream = resp.GetResponseStream())
{
using (StreamReader rdr = new StreamReader(dataStream))
{
json = (String)rdr.ReadToEnd();
rdr.Close();
}
// Close up everything...
dataStream.Close();
}
resp.Close();
}//end using resp
pipe.Send("Api Call complete; Parsing returned data...");
int i = 0;
String h = "";
String l = "";
int s = 0;
int p = 0;
int b = 0;
int payload = 0;
foreach (string line in json.Split(new[] { "}," }, StringSplitOptions.None))
{
if (line != "")
{
l = line;
i = l.Replace("{", "").Length + 1;
p = l.LastIndexOf("{");
if (line.Length > i) //we find this at the beginning of a group of arrays
{
h = line.Substring(0, p - 1);
s = Math.Max(0, h.LastIndexOf("{"));
if (h.Length > s && s != 0)
/*We have a nested array that has more than one level.
*This should only occur at the beginning of new array group.
*Advance the payload counter and get the correct string from line.*/
{
payload++;
l = line.Substring(s, line.Length - s);
}
h = (s >= 0) ? h.Substring(0, s) : h;
//=============
/*At this point 'h' is a nest collection. Split and add to table.*/
string[] OrderedNest = h.Split('{');
for (int z = 0; z < OrderedNest.Length; z++)
{
if (OrderedNest[z] != "")
{
table.Rows.Add(payload, "{" + OrderedNest[z].Replace(":", "").Replace("[","").Replace("]","") + "}");
}
}
//=============
}
else
{
h = null;
}
//at this point the first character in the row should be a "{"; If not we need to add one.
if (l[0].ToString() != "{")
{
l = "{" + l;
}
if (l.Replace("{", "").Length != l.Replace("}", "").Length) //opening and closing braces don't match; match the closing to the opening
{
l = l.Replace("}", "");
b = l.Length - l.Replace("{", "").Length;
l = l + new String('}', b);
}
table.Rows.Add(payload, l.Replace("\\"", "").Replace("\", "").Replace("]","").Replace("[",""));
}
}
//====
using (SqlConnection cnn = new SqlConnection(connectionString))
{
cnn.Open();
using (SqlBulkCopy copy = new SqlBulkCopy(cnn))
{
copy.DestinationTableName = "sp_WebSvcs.dbo.JSON_DATA";
copy.WriteToServer(table);
}
}
//====
} //end try
catch (Exception e)
{
pipe.Send("We have a problem!");
throw new Exception("\n\n" + e.Message + "\n\n");
}
pipe.Send("Parsing complete");
}
}