Javascript / D3.js - 绘制大型数据集 - 提高 d3.js 绘制的 svg 图表中缩放和平移的速度
Javascript / D3.js - draw large data set - improve the speed of zoom and pan in svg chart ploted by d3.js
编辑
刚找到post密谋50 million points with d3.js.
与缩放和平移的交互缓慢是由于 svg 中的元素太多。关键是像 image pyramid. 一样，使用 细节层次 来限制 svg.
中的最大元素
原版post
我正在尝试从 csv/excel 文件中读取一些数据点并使用 d3.js 绘制它们。
数据集包含100,000行，每行包含时间戳和当时的值。
Time stamp, pressure
12/17/2019 12:00:00 AM, 600
我按照 this example 绘制了带有缩放和平移的时间压力图。
没有问题，工作完美。
一个问题是在处理大型数据集时，比如 500,000 个数据点，与图表的交互很慢。
500,000个数据点的图表显示的是整体形状，放大后细节才会显现出来。
放大时，所有数据点都被重新绘制并被裁剪路径裁剪掉。速度还有提升的空间吗？
更新代码
function draw(res){
//clear the current content in the div
document.getElementById("spectrum-fig").innerHTML = '';
var fullwidth = d3.select('#spectrum-fig').node().getBoundingClientRect().width;
fullwidth = fullwidth < 500? 500:fullwidth;
var fullheight = 500;
var resLevelOne = getWindowed(res, 1);
var resLevelTwo = getWindowed(res, 2);
var designMax= getMaxPressureKPa();
var resMax = getPsiTopTen(res);
const SMYSKPa = getSMYSPressureKPa();
const avePsi = getAvePsi(res);
var psiRange = d3.max(res, d=>d.psi) - d3.min(res, d=>d.psi);
var resSmallChart = getWindowed(res, 2);//
//filtSpectrum(res, 0.05*psiRange); //0.05 magic numbers
//var resSmallChart = res;
//margin for focus chart, margin for small chart
var margin = {left:50, right: 50, top: 30, bottom:170},
margin2 = {left:50, right: 50, top: 360, bottom:30},
width = fullwidth - margin.left - margin.right,
height = fullheight - margin.top - margin.bottom,
height2 = fullheight - margin2.top-margin2.bottom;
//x, y, for big chart, x2, y2 for small chart
var x = d3.scaleTime().domain(d3.extent(res, d => d.Time)).range([0, width]),
x2 = d3.scaleTime().domain(d3.extent(res, d => d.Time)).range([0, width]),
y = d3.scaleLinear().domain([0, SMYSKPa]).range([height, 0]),
y2 = d3.scaleLinear().domain([0, SMYSKPa]).range([height2, 0]);
//clear the content in Spectrum-fig div before drawring
//avoid multiple drawings;
var xAxis =d3.axisBottom(x).tickFormat(d3.timeFormat("%m-%d")),
xAxis2 = d3.axisBottom(x2).tickFormat(d3.timeFormat("%b")),
yAxis = d3.axisLeft(y);
var brush = d3.brushX() // Add the brush feature using the d3.brush function
.extent( [ [0,0], [width,height2] ] ) // initialise the brush area: start at 0,0 and finishes at width,height: it means I select the whole graph area
.on("brush end", brushed); // trigger the brushed function
var zoom = d3.zoom()
.scaleExtent([1, 100]) //defined the scale extend
.translateExtent([[0, 0], [width, height]])
.extent([[0, 0], [width, height]])
.on("zoom", zoomed); //at the zoom end trigger zoomed function
//line for big chart line
var line = d3.line()
.x(function(d) { return x(d.Time) })
.y(function(d) { return y(d.psi) });
//line2 for small chart line
var line2 = d3.line()
.x(function(d) { return x2(d.Time) })
.y(function(d) { return y2(d.psi) });
var svg = d3.select("#spectrum-fig")
.append("svg")
.attr("width", fullwidth)
.attr("height", fullheight);
svg.append("defs").append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
var focus = svg.append("g")
.attr("class", "focus")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
var context = svg.append("g")
.attr("class", "context")
.attr("transform", "translate(" + margin2.left + "," + margin2.top + ")");
focus.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate (0," + height +")")
.call(xAxis);
focus.append("g")
.attr("class", "axis axis--y")
.call(yAxis);
focus.append("g")
.attr("transform", "translate (" + width + ", 0)")
.call(d3.axisRight(y).tickFormat('').tickSize(0));
focus.append("g")
.attr("transform", "translate (0, 0)")
.call(d3.axisTop(x).tickFormat('').tickSize(0));
// Add the line
focus.insert("path")
//.datum(res)
.attr("class", "line") // I add the class line to be able to modify this line later on.
.attr("fill", "none")
.attr('clip-path', 'url(#clip)')
.attr("stroke", "steelblue")
.attr("stroke-width", 1.5)
.attr("d", line(resLevelTwo));
context.insert("path")
//.datum(resSmallChart)
.attr("class", "line")
.attr("stroke", "steelblue")
.attr("stroke-width", 1.5)
.attr("fill", "none")
.attr("d", line2(resSmallChart));
context.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height2 + ")")
.call(xAxis2);
context.append("g")
.attr("class", "brush")
.call(brush)
.call(brush.move, x.range());
svg.append("rect")
.attr("class", "zoom")
.attr('fill', 'none')
.attr('cursor', 'move')
.attr('pointer-events', 'all')
.attr("width", width)
.attr("height", height)
.attr("transform", "translate(" + margin.left + "," + margin.top + ")")
.call(zoom);
function getWindowed(arr, level){
var windowed = new Array();
var arrLength = arr.length;
var windowSize =Math.pow(16, level); //set the window size
for(let i = 0; i * windowSize < arrLength; i++ ){ //each to be the window size
let startIndex = i * windowSize;
let endIndex = (i+1) * windowSize;
endIndex = endIndex >= arrLength ? arrLength-1 : endIndex;
let localExtreme = findLocalExtreme(arr.slice(startIndex, endIndex));
if (localExtreme.Max.Time.getTime() === localExtreme.Min.Time.getTime()){ //anything include = need getTime
windowed.push(localExtreme.Max)
}else if(localExtreme.Max.Time < localExtreme.Min.Time){
windowed.push(localExtreme.Max);
windowed.push(localExtreme.Min);
}else{
windowed.push(localExtreme.Min);
windowed.push(localExtreme.Max);
}
}
let firstElement = {...arr[0]};
let lastElement = {...arr[arr.length-1]};
if(firstElement.Time.getTime() != windowed[0].Time.getTime()){ //insert to the position zero
windowed.unshift(firstElement);
}
if(lastElement.Time.getTime() != windowed[windowed.length-1].Time.getTime()){
windowed.push(lastElement);
}//insert to the end last member;
return windowed;
}
function findLocalExtreme(slicedArr){
if(slicedArr === undefined || slicedArr.length == 0){
throw 'error: no array members';
}
let slicedArrLength = slicedArr.length;
let tempMax = {...slicedArr[0]};
let tempMin = {...slicedArr[0]};
if(slicedArrLength === 1){
return {
Max: tempMax,
Min: tempMin
}
}
for (let i = 1; i < slicedArrLength; i++){
if (slicedArr[i].psi > tempMax.psi){
tempMax = {...slicedArr[i]};
}
if (slicedArr[i].psi < tempMin.psi){
tempMin = {...slicedArr[i]};
}
}
return {
Max: tempMax,
Min: tempMin
}
}
function getDataToDraw(timeRange){ //timeRange [0,1] , [startTime, endTime]
const bisect = d3.bisector(d => d.Time).left;
const startIndex = bisect(res, timeRange[0]);
const endIndex = bisect(res, timeRange[1]);
const numberInOriginal = endIndex-startIndex+1;
const windowSize =16;
const maxNumber = 8000;
let level = Math.ceil(Math.log(numberInOriginal/maxNumber ) / Math.log(windowSize));
if(level <=0 ) level =0;
console.log(endIndex, startIndex, endIndex-startIndex+1, level);
if(level === 0){
return res.slice(startIndex, endIndex);
}if(level === 1){
let start_i = bisect(resLevelOne, timeRange[0]);
let end_i =bisect(resLevelOne, timeRange[1]);
return resLevelOne.slice(start_i, end_i);
}else { //if level 2 or higher, never happen
let start_i = bisect(resLevelTwo, timeRange[0]);
let end_i =bisect(resLevelTwo, timeRange[1]);
return resLevelTwo.slice(start_i, end_i);
}
}
function brushed() {
if (d3.event.sourceEvent && d3.event.sourceEvent.type === "zoom") return; // ignore brush-by-zoom
var s = d3.event.selection || x2.range();
x.domain(s.map(x2.invert, x2));
focus.select(".line").attr("d", line(getDataToDraw(x.domain())));
focus.select(".axis--x").call(xAxis);
svg.select(".zoom").call(zoom.transform, d3.zoomIdentity
.scale(width / (s[1] - s[0]))
.translate(-s[0], 0));
}
function zoomed() {
if (d3.event.sourceEvent && d3.event.sourceEvent.type === "brush") return; // ignore zoom-by-brush
var t = d3.event.transform;
//console.log(t);
x.domain(t.rescaleX(x2).domain());
focus.select(".line").attr("d", line(getDataToDraw(t.rescaleX(x2).domain())));
focus.select(".axis--x").call(xAxis);
context.select(".brush").call(brush.move, x.range().map(t.invertX, t));
}
}
这是我的想法。
重新绘制似乎是必须的，因为当你放大点时你怎么能期望有相同的位置？
不过，您可以控制重绘的频率。例如，人们使用 debounce 将任何事件期间的触发次数减少到 50 毫秒以下（例如 pan 尤其如此）。 Debounce 是一个通用的解决方案，您可以检查 lodash 库的一些实现。
.on("zoom", debounced(zoomed)) // lower the chance if you get 5 calls under 500ms
此外，如果涉及任何动画，您可以将动画延迟到缩放（或平移）的最后阶段，这类似于去抖动概念。或者只是简单地禁用动画。
注意：React 确实支持另一种称为并发的模式，默认情况下并未启用，目前还没有。然而它所做的是，假设每个图都由一个小组件捕获，并且它花费 1ms 进行渲染，然后在渲染 16 个组件后，它认为它在这个渲染上花费了太多时间，并将响应返回给浏览器处理其他事情，例如。用户输入等。这样您就可以开始滚动页面或移动鼠标。在下一个周期中，它可以拾取接下来的 16 个组件。假设你有 1000 个组件，它需要几个周期才能完成所有渲染。如果您在中间再次放大，它将跳过前 16 个组件并再次移动到新的渲染。希望你明白了。它可能会帮助您解决最新的 React 18 问题。
参考postplotting 50 million points with d3.js.
与缩放和平移的交互缓慢是由于 svg 中的元素太多。关键是使用细节层次，来限制svg中的最大元素。
编辑
刚找到post密谋50 million points with d3.js.
与缩放和平移的交互缓慢是由于 svg 中的元素太多。关键是像 image pyramid. 一样，使用 细节层次 来限制 svg.
中的最大元素原版post
我正在尝试从 csv/excel 文件中读取一些数据点并使用 d3.js 绘制它们。
数据集包含100,000行，每行包含时间戳和当时的值。
Time stamp, pressure
12/17/2019 12:00:00 AM, 600
我按照 this example 绘制了带有缩放和平移的时间压力图。
没有问题，工作完美。
一个问题是在处理大型数据集时，比如 500,000 个数据点，与图表的交互很慢。
500,000个数据点的图表显示的是整体形状，放大后细节才会显现出来。
放大时，所有数据点都被重新绘制并被裁剪路径裁剪掉。速度还有提升的空间吗？
更新代码
function draw(res){
//clear the current content in the div
document.getElementById("spectrum-fig").innerHTML = '';
var fullwidth = d3.select('#spectrum-fig').node().getBoundingClientRect().width;
fullwidth = fullwidth < 500? 500:fullwidth;
var fullheight = 500;
var resLevelOne = getWindowed(res, 1);
var resLevelTwo = getWindowed(res, 2);
var designMax= getMaxPressureKPa();
var resMax = getPsiTopTen(res);
const SMYSKPa = getSMYSPressureKPa();
const avePsi = getAvePsi(res);
var psiRange = d3.max(res, d=>d.psi) - d3.min(res, d=>d.psi);
var resSmallChart = getWindowed(res, 2);//
//filtSpectrum(res, 0.05*psiRange); //0.05 magic numbers
//var resSmallChart = res;
//margin for focus chart, margin for small chart
var margin = {left:50, right: 50, top: 30, bottom:170},
margin2 = {left:50, right: 50, top: 360, bottom:30},
width = fullwidth - margin.left - margin.right,
height = fullheight - margin.top - margin.bottom,
height2 = fullheight - margin2.top-margin2.bottom;
//x, y, for big chart, x2, y2 for small chart
var x = d3.scaleTime().domain(d3.extent(res, d => d.Time)).range([0, width]),
x2 = d3.scaleTime().domain(d3.extent(res, d => d.Time)).range([0, width]),
y = d3.scaleLinear().domain([0, SMYSKPa]).range([height, 0]),
y2 = d3.scaleLinear().domain([0, SMYSKPa]).range([height2, 0]);
//clear the content in Spectrum-fig div before drawring
//avoid multiple drawings;
var xAxis =d3.axisBottom(x).tickFormat(d3.timeFormat("%m-%d")),
xAxis2 = d3.axisBottom(x2).tickFormat(d3.timeFormat("%b")),
yAxis = d3.axisLeft(y);
var brush = d3.brushX() // Add the brush feature using the d3.brush function
.extent( [ [0,0], [width,height2] ] ) // initialise the brush area: start at 0,0 and finishes at width,height: it means I select the whole graph area
.on("brush end", brushed); // trigger the brushed function
var zoom = d3.zoom()
.scaleExtent([1, 100]) //defined the scale extend
.translateExtent([[0, 0], [width, height]])
.extent([[0, 0], [width, height]])
.on("zoom", zoomed); //at the zoom end trigger zoomed function
//line for big chart line
var line = d3.line()
.x(function(d) { return x(d.Time) })
.y(function(d) { return y(d.psi) });
//line2 for small chart line
var line2 = d3.line()
.x(function(d) { return x2(d.Time) })
.y(function(d) { return y2(d.psi) });
var svg = d3.select("#spectrum-fig")
.append("svg")
.attr("width", fullwidth)
.attr("height", fullheight);
svg.append("defs").append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
var focus = svg.append("g")
.attr("class", "focus")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
var context = svg.append("g")
.attr("class", "context")
.attr("transform", "translate(" + margin2.left + "," + margin2.top + ")");
focus.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate (0," + height +")")
.call(xAxis);
focus.append("g")
.attr("class", "axis axis--y")
.call(yAxis);
focus.append("g")
.attr("transform", "translate (" + width + ", 0)")
.call(d3.axisRight(y).tickFormat('').tickSize(0));
focus.append("g")
.attr("transform", "translate (0, 0)")
.call(d3.axisTop(x).tickFormat('').tickSize(0));
// Add the line
focus.insert("path")
//.datum(res)
.attr("class", "line") // I add the class line to be able to modify this line later on.
.attr("fill", "none")
.attr('clip-path', 'url(#clip)')
.attr("stroke", "steelblue")
.attr("stroke-width", 1.5)
.attr("d", line(resLevelTwo));
context.insert("path")
//.datum(resSmallChart)
.attr("class", "line")
.attr("stroke", "steelblue")
.attr("stroke-width", 1.5)
.attr("fill", "none")
.attr("d", line2(resSmallChart));
context.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height2 + ")")
.call(xAxis2);
context.append("g")
.attr("class", "brush")
.call(brush)
.call(brush.move, x.range());
svg.append("rect")
.attr("class", "zoom")
.attr('fill', 'none')
.attr('cursor', 'move')
.attr('pointer-events', 'all')
.attr("width", width)
.attr("height", height)
.attr("transform", "translate(" + margin.left + "," + margin.top + ")")
.call(zoom);
function getWindowed(arr, level){
var windowed = new Array();
var arrLength = arr.length;
var windowSize =Math.pow(16, level); //set the window size
for(let i = 0; i * windowSize < arrLength; i++ ){ //each to be the window size
let startIndex = i * windowSize;
let endIndex = (i+1) * windowSize;
endIndex = endIndex >= arrLength ? arrLength-1 : endIndex;
let localExtreme = findLocalExtreme(arr.slice(startIndex, endIndex));
if (localExtreme.Max.Time.getTime() === localExtreme.Min.Time.getTime()){ //anything include = need getTime
windowed.push(localExtreme.Max)
}else if(localExtreme.Max.Time < localExtreme.Min.Time){
windowed.push(localExtreme.Max);
windowed.push(localExtreme.Min);
}else{
windowed.push(localExtreme.Min);
windowed.push(localExtreme.Max);
}
}
let firstElement = {...arr[0]};
let lastElement = {...arr[arr.length-1]};
if(firstElement.Time.getTime() != windowed[0].Time.getTime()){ //insert to the position zero
windowed.unshift(firstElement);
}
if(lastElement.Time.getTime() != windowed[windowed.length-1].Time.getTime()){
windowed.push(lastElement);
}//insert to the end last member;
return windowed;
}
function findLocalExtreme(slicedArr){
if(slicedArr === undefined || slicedArr.length == 0){
throw 'error: no array members';
}
let slicedArrLength = slicedArr.length;
let tempMax = {...slicedArr[0]};
let tempMin = {...slicedArr[0]};
if(slicedArrLength === 1){
return {
Max: tempMax,
Min: tempMin
}
}
for (let i = 1; i < slicedArrLength; i++){
if (slicedArr[i].psi > tempMax.psi){
tempMax = {...slicedArr[i]};
}
if (slicedArr[i].psi < tempMin.psi){
tempMin = {...slicedArr[i]};
}
}
return {
Max: tempMax,
Min: tempMin
}
}
function getDataToDraw(timeRange){ //timeRange [0,1] , [startTime, endTime]
const bisect = d3.bisector(d => d.Time).left;
const startIndex = bisect(res, timeRange[0]);
const endIndex = bisect(res, timeRange[1]);
const numberInOriginal = endIndex-startIndex+1;
const windowSize =16;
const maxNumber = 8000;
let level = Math.ceil(Math.log(numberInOriginal/maxNumber ) / Math.log(windowSize));
if(level <=0 ) level =0;
console.log(endIndex, startIndex, endIndex-startIndex+1, level);
if(level === 0){
return res.slice(startIndex, endIndex);
}if(level === 1){
let start_i = bisect(resLevelOne, timeRange[0]);
let end_i =bisect(resLevelOne, timeRange[1]);
return resLevelOne.slice(start_i, end_i);
}else { //if level 2 or higher, never happen
let start_i = bisect(resLevelTwo, timeRange[0]);
let end_i =bisect(resLevelTwo, timeRange[1]);
return resLevelTwo.slice(start_i, end_i);
}
}
function brushed() {
if (d3.event.sourceEvent && d3.event.sourceEvent.type === "zoom") return; // ignore brush-by-zoom
var s = d3.event.selection || x2.range();
x.domain(s.map(x2.invert, x2));
focus.select(".line").attr("d", line(getDataToDraw(x.domain())));
focus.select(".axis--x").call(xAxis);
svg.select(".zoom").call(zoom.transform, d3.zoomIdentity
.scale(width / (s[1] - s[0]))
.translate(-s[0], 0));
}
function zoomed() {
if (d3.event.sourceEvent && d3.event.sourceEvent.type === "brush") return; // ignore zoom-by-brush
var t = d3.event.transform;
//console.log(t);
x.domain(t.rescaleX(x2).domain());
focus.select(".line").attr("d", line(getDataToDraw(t.rescaleX(x2).domain())));
focus.select(".axis--x").call(xAxis);
context.select(".brush").call(brush.move, x.range().map(t.invertX, t));
}
}
这是我的想法。
重新绘制似乎是必须的，因为当你放大点时你怎么能期望有相同的位置？
不过，您可以控制重绘的频率。例如，人们使用 debounce 将任何事件期间的触发次数减少到 50 毫秒以下（例如 pan 尤其如此）。 Debounce 是一个通用的解决方案，您可以检查 lodash 库的一些实现。
.on("zoom", debounced(zoomed)) // lower the chance if you get 5 calls under 500ms
此外，如果涉及任何动画，您可以将动画延迟到缩放（或平移）的最后阶段，这类似于去抖动概念。或者只是简单地禁用动画。
注意：React 确实支持另一种称为并发的模式，默认情况下并未启用，目前还没有。然而它所做的是，假设每个图都由一个小组件捕获，并且它花费 1ms 进行渲染，然后在渲染 16 个组件后，它认为它在这个渲染上花费了太多时间，并将响应返回给浏览器处理其他事情，例如。用户输入等。这样您就可以开始滚动页面或移动鼠标。在下一个周期中，它可以拾取接下来的 16 个组件。假设你有 1000 个组件，它需要几个周期才能完成所有渲染。如果您在中间再次放大，它将跳过前 16 个组件并再次移动到新的渲染。希望你明白了。它可能会帮助您解决最新的 React 18 问题。
参考postplotting 50 million points with d3.js.
与缩放和平移的交互缓慢是由于 svg 中的元素太多。关键是使用细节层次，来限制svg中的最大元素。