OSMServer/OsmXmlToRegionConverter/Manager.cs

using System.Globalization;
using System.Xml;

namespace OsmXmlToRegionConverter;

public class Manager
{
    private static readonly XmlReaderSettings readerSettings = new()
    {
        IgnoreWhitespace = true,
        IgnoreComments = true
    };
    
    private static readonly NumberFormatInfo decimalInfo = new()
    {
        NumberDecimalSeparator = "."
    };

    private const string NodesTmpFilename = "nodes.tmp";
    private const string WaysTmpFilename = "ways.tmp";
    private const string TagsFilename = "tags";
    
    public static void XmlToByteConverter(string inputFilePath, string outputFolderPath)
    {
        if (!File.Exists(inputFilePath))
            throw new FileNotFoundException(inputFilePath);
        Directory.CreateDirectory(outputFolderPath);
        
        Console.WriteLine("Getting highwayNodeIDs...");
        FileStream xmlFileStream = new FileStream(inputFilePath, FileMode.Open);
        HashSet<ulong> nodeIdsToImport = GetHighwayNodeIds(XmlReader.Create(xmlFileStream, readerSettings));
        xmlFileStream.Position = 0;
        
        Console.WriteLine("Converting Nodes...");
        Dictionary<ulong, ulong> nodeRegions = ImportNodesToRegionsTmp(XmlReader.Create(xmlFileStream, readerSettings), nodeIdsToImport, outputFolderPath);
        nodeIdsToImport.Clear(); //memory management
        xmlFileStream.Position = 0;

        Console.WriteLine("Converting Ways...");
        ImportWaysToRegionTmp(XmlReader.Create(xmlFileStream, readerSettings), nodeRegions, outputFolderPath);
        HashSet<ulong> regionHashes = nodeRegions.Values.ToHashSet();
        nodeRegions.Clear(); //memory management
        
        Console.WriteLine("Converting Nodes and Ways to Nodes and Edges in regionfile...");
        ConvertNodesAndWaysToRegion(outputFolderPath, regionHashes);
    }
    
    private static HashSet<ulong> GetHighwayNodeIds(XmlReader xmlReader)
    {
        HashSet<ulong> retSet = new();
        bool isHighway;
        HashSet<ulong> currentIds = new();
        while (xmlReader.ReadToFollowing("way"))
        {
            isHighway = false;
            currentIds.Clear();
            XmlReader wayReader = xmlReader.ReadSubtree();
            while (wayReader.Read())
            {
                if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway"))
                {
                    isHighway = true;
                }
                else if (xmlReader.Name == "nd")
                {
                    try
                    {
                        currentIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
                    }
                    catch (FormatException) { };
                }
            }
            if (isHighway)
            {
                retSet.UnionWith(currentIds);
            }
            wayReader.Close();
        }
        xmlReader.Close();

        return retSet;
    }

    private static Dictionary<ulong, ulong> ImportNodesToRegionsTmp(XmlReader xmlReader, HashSet<ulong> nodeIdsToImport, string outputFolderPath)
    {
        Dictionary<ulong, FileStream> regionWriters = new Dictionary<ulong, FileStream>();
        Dictionary<ulong, ulong> nodeRegions = new Dictionary<ulong, ulong>();

        while (xmlReader.ReadToFollowing("node"))
        {
            ulong currentNodeId = Convert.ToUInt64(xmlReader.GetAttribute("id"));
            if (nodeIdsToImport.Contains(currentNodeId))
            {
                float lat = Convert.ToSingle(xmlReader.GetAttribute("lat")!, decimalInfo);
                float lon = Convert.ToSingle(xmlReader.GetAttribute("lon")!, decimalInfo);
                TmpNode newTmpNode = new TmpNode(currentNodeId, lat, lon);
                ulong regionHash = Coordinates.GetRegionHashCode(newTmpNode.coordinates);

                if (!regionWriters.ContainsKey(regionHash))
                {
                    Directory.CreateDirectory(Path.Combine(outputFolderPath, regionHash.ToString()));
                    regionWriters.Add(regionHash, new FileStream(Path.Combine(outputFolderPath, regionHash.ToString(), NodesTmpFilename), FileMode.Append));
                }
                regionWriters[regionHash].Write( ByteConverter.GetBytes(newTmpNode));
                nodeRegions.Add(currentNodeId, regionHash);
            }
        }
        foreach(FileStream regionWriter in regionWriters.Values)
            regionWriter.Close();
        xmlReader.Close();
        return nodeRegions;
    }

    private static void ImportWaysToRegionTmp(XmlReader xmlReader, Dictionary<ulong, ulong> nodeRegions, string outputFolderPath)
    {
        Dictionary<ulong, FileStream> regionWriters = new Dictionary<ulong, FileStream>();
        bool isHighway = false;
        HashSet<ulong> currentIds = new();
        while (xmlReader.ReadToFollowing("way"))
        {
            HashSet<Tag> tagsSet = new HashSet<Tag>();
            isHighway = false;
            currentIds.Clear();
            ulong wayId = Convert.ToUInt64(xmlReader.GetAttribute("id"));
            XmlReader wayReader = xmlReader.ReadSubtree();
            while (wayReader.Read())
            {
                if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway"))
                {
                    isHighway = true;
                }
                else if (xmlReader.Name == "nd")
                {
                    try
                    {
                        currentIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
                    }
                    catch (FormatException) { };
                }else if (xmlReader.Name == "tag")
                {
                    tagsSet.Add(Tag.ConvertToTag(xmlReader.GetAttribute("k")!, xmlReader.GetAttribute("v")!));
                }
            }
            if (isHighway)
            {
                TmpWay newTmpWay = new TmpWay(wayId, currentIds.ToArray(), tagsSet);
                HashSet<ulong> regionsToWrite = new HashSet<ulong>();
                foreach (ulong nodeId in currentIds)
                {
                    if (nodeRegions.ContainsKey(nodeId))
                    {
                        regionsToWrite.Add(nodeRegions[nodeId]);
                    }
                }
                

                foreach (ulong regionHash in regionsToWrite)
                {
                    if (!regionWriters.ContainsKey(regionHash))
                    {
                        regionWriters.Add(regionHash, new FileStream(Path.Combine(outputFolderPath, regionHash.ToString(), WaysTmpFilename), FileMode.Append));
                    }
                    regionWriters[regionHash].Write(ByteConverter.GetBytes(newTmpWay));
                }
            }
            wayReader.Close();
        }
        foreach(FileStream regionWriter in regionWriters.Values)
            regionWriter.Close();
        xmlReader.Close();
    }

    private static void ConvertNodesAndWaysToRegion(string outputFolderPath, HashSet<ulong> regionHashes)
    {
        foreach (ulong regionHash in regionHashes)
        {
            Console.WriteLine($"Converting Region {regionHash}");
            string regionFilePath = Path.Combine(outputFolderPath, $"{regionHash.ToString()}.region");
            string regionFolderPath = Path.Combine(outputFolderPath, regionHash.ToString());
            string nodesFilePath = Path.Combine(regionFolderPath, NodesTmpFilename);
            string waysFilePath = Path.Combine(regionFolderPath, WaysTmpFilename);

            Region newRegion = new Region(regionHash);
            Dictionary<ulong, Node> nodes = new Dictionary<ulong, Node>();

            byte[] nodeBytes = File.ReadAllBytes(nodesFilePath);
            HashSet<object> nodeObjects =  ByteConverter.GetObjectsFromBytes(nodeBytes);
            foreach (object obj in nodeObjects)
            {
                if (obj.GetType() == typeof(TmpNode))
                {
                    TmpNode tmpNode = (TmpNode)obj;
                    Node newNode = new Node(tmpNode.id, tmpNode.coordinates);
                    newRegion.AddNode(newNode);
                    nodes.TryAdd(newNode.id, newNode);
                }
                else
                    throw new Exception("Unexpected object.");
            }

            byte[] wayBytes = File.ReadAllBytes(waysFilePath);
            HashSet<object> ways = ByteConverter.GetObjectsFromBytes(wayBytes);
            foreach (object obj in ways)
            {
                if (obj.GetType() == typeof(TmpWay))
                {
                    TmpWay way = (TmpWay)obj;
                    for (int nodeIdi = 0; nodeIdi < way.wayNodeIds.Length - 1; nodeIdi++) //TODO directional
                    {
                        if (nodes.ContainsKey(way.wayNodeIds[nodeIdi]) &&
                            nodes.ContainsKey(way.wayNodeIds[nodeIdi + 1]))
                        {
                            Node n1 = nodes[way.wayNodeIds[nodeIdi]];
                            Node n2 = nodes[way.wayNodeIds[nodeIdi + 1]];
                            n1.edges.Add(new Edge(way.id, n2));
                            n2.edges.Add(new Edge(way.id, n1));
                        }
                    }
                }
                else throw new Exception("Unexpected object.");
            }

            File.WriteAllBytes(regionFilePath, ByteConverter.GetBytes(newRegion));
            Directory.Delete(regionFolderPath, true);
        }
        
    }
}