D3.JS time-series line chart with real-time data, panning and zooming

Question

FIDDLE <<<< this has more up to date code than in the question.

I am trying to create a real-time (live-updating) time-series chart in d3, that can also be panned (in X) and zoomed. Ideally the functionality that I want is if the the right-most part of the line is visible to the user, then when new data is added to the graph, it will pan sideways automatically to include the new data (without changing axis scales).

My d3.json() requests should return JSON arrays that look like this :

[{"timestamp":1399325270,"value":-0.0029460209892230222598710528},{"timestamp":1399325271,"value":-0.0029460209892230222598710528},{"timestamp":1399325279,"value":-0.0029460209892230222598710528},....]

When the page first loads, I make a request and get all the available date up to now, and draw the graph - easy. The following code does this, it also allows panning (in X) and zooming.

var globalData;
var lastUpdateTime = "0";
var dataIntervals = 1;

var margin = { top: 20, right: 20, bottom: 30, left: 50 },
    width = document.getElementById("chartArea").offsetWidth - margin.left - margin.right,
    height = document.getElementById("chartArea").offsetHeight - margin.top - margin.bottom;

var x = d3.time.scale()
    .range([0, width]);

var y = d3.scale.linear()
    .range([height, 0]);

var xAxis = d3.svg.axis()
    .scale(x)
    .orient("bottom")
    .ticks(10)
    .tickFormat(d3.time.format('%X'))
    .tickSize(1);
    //.tickPadding(8);

var yAxis = d3.svg.axis()
    .scale(y)
    .orient("left");

var valueline = d3.svg.line()
    .x(function (d) { return x(d.timestamp); })
    .y(function (d) { return y(d.value); });

var zoom = d3.behavior.zoom()
    .x(x)
    .y(y)
    .scaleExtent([1, 4])
    .on("zoom", zoomed);

var svg = d3.select("#chartArea")
    .append("svg")
    .attr("width", width + margin.left + margin.right)
    .attr("height", height + margin.top + margin.bottom)
    .append("g")
    .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
    .call(zoom);

svg.append("rect")
    .attr("width", width)
    .attr("height", height)
    .attr("class", "plot"); // ????

var clip = svg.append("clipPath")
    .attr("id", "clip")
    .append("rect")
    .attr("x", 0)
    .attr("y", 0)
    .attr("width", width)
    .attr("height", height);

var chartBody = svg.append("g")
    .attr("clip-path", "url(#clip)");

svg.append("g")         // Add the X Axis
    .attr("class", "x axis")
    .attr("transform", "translate(0," + height + ")")
    .call(xAxis);

svg.append("g")         // Add the Y Axis
    .attr("class", "y axis")
    .call(yAxis);

svg.append("text")
    .attr("transform", "rotate(-90)")
    .attr("y", 0 - margin.left)
    .attr("x", (0 - (height / 2)))
    .attr("dy", "1em")
    .style("text-anchor", "middle")
    .text("Return (%)");

// plot the original data by retrieving everything from time 0
d3.json("/performance/benchmark/date/0/interval/" + dataIntervals, function (error, data) {
    data.forEach(function (d) {

        lastUpdateTime = String(d.timestamp); // this will be called until the last element and so will have the value of the last element
        d.timestamp = new Date(d.timestamp);
        d.value = d.value * 100;

    });

    globalData = data;

    x.domain(d3.extent(globalData, function (d) { return d.timestamp; }));
    y.domain(d3.extent(globalData, function (d) { return d.value; }));

    chartBody.append("path")        // Add the valueline path
        .datum(globalData)
        .attr("class", "line")
        .attr("d", valueline);

    var inter = setInterval(function () {
        updateData();
    }, 5000);

});

var panMeasure = 0;
var oldScale = 1;
function zoomed() {
    //onsole.log(d3.event);
    d3.event.translate[1] = 0;
    svg.select(".x.axis").call(xAxis);

    if (Math.abs(oldScale - d3.event.scale) > 1e-5) {
        oldScale = d3.event.scale;
        svg.select(".y.axis").call(yAxis);
    }

    svg.select("path.line").attr("transform", "translate(" + d3.event.translate[0] + ",0)scale(" + d3.event.scale + ", 1)");
    panMeasure = d3.event.translate[0];

}

In the following block of code, I make a http request to get all the new data and add this to the chart. This works fine. Now I just need to sort out the pan logic for the new data - which I imagine would go in here:

var dx = 0;
function updateData() {

    var newData = [];

        d3.json("/performance/benchmark/date/" + lastUpdateTime + "/interval/" + dataIntervals, function (error, data) {
            data.forEach(function (d) {

                lastUpdateTime = String(d.timestamp); // must be called before its converted to Date()
                d.timestamp = new Date(d.timestamp);
                d.value = d.value * 100;

                globalData.push(d);
                newData.push(d);

            });

            // panMeasure would be some measure of how much the user has panned (ie if the right-most part of the graph is still visible to the user.
            if (panMeasure <= 0) { // add the new data and pan

                x1 = newData[0].timestamp;
                x2 = newData[newData.length - 1].timestamp;
                dx = dx + (x(x1) - x(x2)); // dx needs to be cummulative

                d3.select("path")
                    .datum(globalData)
                    .attr("class", "line")
                    .attr("d", valueline(globalData))
                .transition()
                    .ease("linear")
                    .attr("transform", "translate(" + String(dx) + ")");

            }

            else { // otherwise - just add the new data 
                d3.select("path")
                    .datum(globalData)
                    .attr("class", "line")
                    .attr("d", valueline(globalData));
            }

            svg.select(".x.axis").call(xAxis);

        });
}

What I'm trying to do (I guess that's what I should be doing) is get the range of the time values for the new data (ie the difference between the first value and the last value of the newData[] array, convert this to pixels and then pan the line using this number.

This seems to sort of work, but only on the first update. Another problem is if I do any panning/zooming using the mouse while the data is trying to be updated, the line disappears and doesn't necessarily come back on the next update. I would really appreciate some feedback on potential errors you can spot in the code and/or how this should be done. Thanks.

UPDATE 1:

Okay, so I have figured out what the problem with the automatic panning was. I realised that the translate vector needs to have a cumulative value from some origin, so once I made dx cumulative (dx = dx + (x(x2) - x(x1)); then the sideways panning started working for when new data was added.

UPDATE 2:

I have now included a fiddle that is close to the actual way I expect data to be retrieved and plotted. It seems to work to some extent the way I want it except for:

1. X-axis tick marks don't pan with the new data
2. When I do manual pan, the behaviour becomes a bit strange on the first pan (jumps back a bit)
3. If I am panning/zooming when the new data is trying to be added - the line disappears ('multi-threading' issues? :S)

Answer 1

Instead of plotting the whole data and clipping the unnecessary parts, you could keep a global array of all values that you just slice each time you need to update the graph. Then you can more easily recompute your x axis & values. Downside is you cannot easily have a transition.

So, you would have two variables:

var globalOffset = 0;
var panOffset = 0;

globalOffset would be updated each time you populate new values and panOffset each time you pan. Then you just slice your data before plotting:

var offset = Math.max(0, globalOffset + panOffset);
var graphData = globalData.slice(offset, offset + 100);

See fiddle for complete solution.