how to walk async recursively over an object, doing work (in right order) per leaf

Hi.

My opensourced htmlMicroscope works well, except when you open a
sub-array that holds more than say a hundred keys. Maybe 200.

But the thing is, I'd like it to work on all kinds of arrays,
including ones that have 1500 keys in some sub-object/array somewhere.

Like for instance my serviceLog now does when viewing the hits per
month (not per day).

And I'm sure other applications that might use my htmlMicroscope will
suffer suddenly from the same problem, forcing the end-user of hm() to
mold his/her data at unexpected turns, and sometimes against his/her
wishes.

So I'd like to solve this issue. Please help me out here, I've been
stuck on it for the better part of a day now..


How do you split this loop up so that it runs asynchronously?
(it would need to return a placeholder HTML with unique id= instantly, and would replace the placeholder with the actual content when done)

function printLevel : function (data) {
  var html = '';
  if (typeof data=='object' && data!==null && data!==undefined) {
    for (var k in data) {  
      // and the problem is you'd have up to several thousand var k's. the loop needs to setTimeout after say a hundred iterations through THIS loop...
      var v = data[k];
      html += '<tr><td>'+printVariable(k)+'</td><td>'+printVariable (v)+'</td></tr>';
    }
  }
  return html;
}

function printVariable : function (data) {
   var t = typeof data;
   switch (t) {
     case 'number' : 
     case 'string' : 
     case 'boolean':
           return data;
           break;
     case 'object':
         return printLevel (data);
         break;
   }
}

Code (markup):

The real version of this code is somewhat more complicated, and uses a "levelsAtOnce" parameter to not print too many levels deep at the same time. I can work that all in I think.

 

I think I've got it figured out now, folks.

I will async build up a flat list containing all the items from the recursive array that need processing, then process that list async as well, and replace the placeholder with the result when done..

The following code works on a data structure that looks like this;

{
  hmData : {
       mijnDataA : 'a',
       mijnDataB : 'b',
       mijnSubKeyA : {
             hmData : {
                 mijnDataX : 'x'
             },
             hmSettings : {
                // app-specifieke waarden voor het object met 'mijnDataX' als key
             }
       }
  },
  hmSettings : {
     // app-specifieke berekende waarden voor hmData met keys mijnDataA, mijnDataB en mijnSubKeyA
  }
}

Code (markup):

Then, the code that asynchronously builds up the flat worker list :

        printNextLevel : function (pvCmd) {
            if (
                typeof pvCmd.val == 'object' 
                && typeof pvCmd.val.hmStats == 'object' 
                && typeof pvCmd.val.hmData == 'object'
            ) {
                //if (pvCmd.keyValueName && pvCmd.keyValueName!='') pvCmd.val.hmStats.keyValueName=pvCmd.keyValueName; //bit of a hack, i agree.
                var td = typeof pvCmd.val.hmData;
                var d = pvCmd.val.hmData;
            } else {
                var td = typeof pvCmd.val;
                var d = pvCmd.val;
            };
            
            pvCmd.scanPointer = d;
            rajmv.hms.tools.printNextLevel_scan (pvCmd, rajmv.hms.tools.printNextLevel_buildDatanodes);
            
            return {
                html : '<tr><td>Have Yet To Render This</td></tr>'
            };
        },
        
        printNextLevel_scan : function (pvCmd, callback) {
            if (!pvCmd.scanResults) {
                pvCmd.scanResults = [{level:1, datanode:pvCmd.scanPointer}];
                pvCmd.scanIdx = 0;
                pvCmd.scanCount = 0;
                pvCmd.lastPause = 0;
                pvCmd.scanCallback = callback;
            }
            
            rajmv.hms.tools.printNextLevel_scanItem (pvCmd);
            pvCmd.scanCount++;
            
            //rajmv.log (2, 'pvCmd.scanIdx='+pvCmd.scanIdx+', scanResults.length-1='+(pvCmd.scanResults.length-1));
            if (pvCmd.scanIdx==pvCmd.scanResults.length-1) {
                if (typeof pvCmd.scanCallback=='function') {
                    pvCmd.scanCallback (pvCmd);
                } 
                return true; // scanning done!
            }
            
            var pauseFactor = pvCmd.scanCount / 7;
            if (pauseFactor > pvCmd.lastPause + 1) {
                setTimeout (function () {
                    pvCmd.lastPause = pauseFactor;
                    rajmv.hms.tools.printNextLevel_scan(pvCmd);
                }, 50);
            } else {
                rajmv.hms.tools.printNextLevel_scan(pvCmd);
            };
            return false; // not done yet
        },
        
        printNextLevel_scanItem : function (pvCmd) {
            var it = pvCmd.scanResults[pvCmd.scanIdx];
            if (!it || !it.datanode) return false;
            var tit = typeof it.datanode;
            if (tit=='object' && it.datanode!==null && it.datanode!==undefined) {
                if (!it.keys && it.level<=pvCmd.levelsAtOnce) {
                    it.keys = Object.keys (it.datanode);
                    it.keyIdx = 0;
                }
            }
            if (it.keys) {
                if (it.keyIdx<it.keys.length) {
                    var doUntil = it.keyIdx+20;
                    while (it.keyIdx<doUntil && it.keyIdx<it.keys.length-1) {
                        rajmv.hms.tools.printNextLevel_scanKey (pvCmd);
                        it.keyIdx++;
                        pvCmd.scanCount++;
                        
                        if (it.keyIdx==it.keys.length-1) {
                            pvCmd.scanIdx++;
                            pvCmd.scanCount++;
                        }


                        var pauseFactor = pvCmd.scanCount / 7;
                        if (pauseFactor > pvCmd.lastPause + 1) break;
                    }
                }
            } else {
                pvCmd.scanIdx++;
                pvCmd.scanCount++;
            }
        },
        
        printNextLevel_scanKey : function (pvCmd) {
            var it = pvCmd.scanResults[pvCmd.scanIdx];
            if (!it.keys[it.keyIdx]) debugger;
            pvCmd.scanResults.splice(pvCmd.scanIdx+1, 0, {level:it.level+1, datanode:it.datanode[it.keys[it.keyIdx]].hmData}); // insert next datanode just after the scanIdx
            rajmv.log (2, 'hms.tools.printNextLevel_scanKey(): pvCmd.scanCount='+pvCmd.scanCount);
        },
        
        printNextLevel_buildDatanodes : function (pvCmd) {
            if (!pvCmd.html) pvCmd.html = '';
            
            debugger;
            
        },

Code (markup):