O-Calc

Fast indexing of PPLX data – O-Calc Pro Wiki

Written by Admin | Jan 5, 2016 7:15:00 PM

One of the tasks that sometimes comes up during plugin development is the need to rapidly parse and spatially index a large number of PPLX files.

Performing this task by fully hydrating each PPLX file into O-Calc proper, while relatively quick, would still take more time than desired for a large number of poles. In this post is presented a code snippet that can parse the latitude, longitude, poleID, and MCU percentage from a PPLX file very rapidly and allow for the processing of tens to hundreds of PPLX files per second depending on your system’s IO performance.

First I need to create a structure to hold the data I intend to parse form the PPLX and then instantiate a list to hold an instance of that structure for each PPLX that I parse.

  1. public struct Pole
  2. {
  3.     public double cLat ;
  4.     public double cLon ;
  5.     public string cName ;
  6.     public string cPath ;
  7.     public double cMCU ;
  8. }
  9.  
  10. public List <Pole > cPoles = new List <Pole > ( ) ;

Now if I create a list of file path strings of all of the PPLX files I wish to process *the construction of this list is not covered here but typically would involce one or more calls to files = Directory.GetFiles(directory, “*.pplx”);) I can process them as follows:

  1. foreach ( String file in files )
  2. {
  3.     using (StreamReader sr = new StreamReader (file ) )
  4.     {
  5.         using (XmlTextReader xr = new System.Xml . XmlTextReader (sr ) )
  6.         {
  7.             String poleId = System.IO . Path . GetFileNameWithoutExtension (file ) ;
  8.             double lat = 0 ;
  9.             double lon = 0 ;
  10.             while (xr . Read ( ) )
  11.             {
  12.                 if (xr . NodeType == System.Xml . XmlNodeType . Element && xr . Name == "VALUE" )
  13.                 {
  14.                     String sname = xr . GetAttribute ( "NAME" ) ;
  15.                     if (sname == "Latitude" )
  16.                     {
  17.                         xr . Read ( ) ;
  18.                         if ( ! double . TryParse (xr . Value, out lat ) ) break ;
  19.                         if (lat == 0 ) break ;
  20.                     }
  21.                     else if (sname == "Longitude" )
  22.                     {
  23.                         xr . Read ( ) ;
  24.                         if ( ! double . TryParse (xr . Value, out lon ) ) break ;
  25.                         if (lon == 0 ) break ;
  26.  
  27.                         double mcu = - 1 ;
  28.                         try
  29.                         {
  30.                             sr . DiscardBufferedData ( ) ;
  31.                             sr . BaseStream . Seek ( - 1024, SeekOrigin . End ) ;
  32.                             String sbuff = sr . ReadToEnd ( ) ;
  33.                             int idx = sbuff . IndexOf ( "PercentAtMCU" ) ;
  34.                             if (idx > 0 )
  35.                             {
  36.                                 mcu = Parse (sbuff . Substring (idx + 20, 100 ) ) ;
  37.                             }
  38.                         }
  39.                         catch
  40.                         {
  41.                             mcu = - 1 ;
  42.                         }
  43.  
  44.                         if (lat != 0 && lon != 0 )
  45.                         {
  46.                             Pole pole = new Pole ( ) ;
  47.                             pole . cLat = lat ;
  48.                             pole . cLon = lon ;
  49.                             pole . cName = poleId ;
  50.                             pole . cMCU = mcu ;
  51.                             pole . cPath = file ;
  52.                             cPoles . Add (pole ) ;
  53.                         }
  54.                         break ;
  55.  
  56.                     }
  57.                 }
  58.             }
  59.         }
  60.     }
  61. }

Supported by…

  1. double Parse ( String pStr )
  2. {
  3.     const char gt = ( char ) 62 ;
  4.     const char lt = ( char ) 60 ;
  5.     int idx = pStr . IndexOf (gt ) ;
  6.     String s = pStr . Substring (idx + 1 ) ;
  7.     idx = s . IndexOf (lt ) ;
  8.     s = s . Substring ( 0, idx ) ;
  9.     double v = 0 ;
  10.     if ( double . TryParse (s, out v ) ) return v ;
  11.     return 0 ;
  12. }

The following video shows this code in operation…