using System.Globalization; using System.Runtime.Serialization.Formatters.Binary; using System.Text.Json; using System.Xml; using OSMDatastructure; using OSMDatastructure.Graph; namespace Server; public class RegionConverter { private static readonly XmlReaderSettings ReaderSettings = new() { IgnoreWhitespace = true, IgnoreComments = true }; private static readonly NumberFormatInfo decimalInfo = new() { NumberDecimalSeparator = "." }; public const string NodesFileName = "region.nodes"; public const string WaysFileName = "region.ways"; public const string TagsFileName = "region.tags"; public const string RegionsFileName = ".region"; public static void ConvertXMLToRegions(string filePath, string outputPath) { if (!File.Exists(filePath)) throw new FileNotFoundException(); if (!Directory.Exists(outputPath)) Directory.CreateDirectory(outputPath); Console.WriteLine("Converting Nodes..."); FileStream xmlFileStream = new FileStream(filePath, FileMode.Open); Dictionary nodeIdRegionDict = GetNodesAndRegions(XmlReader.Create(xmlFileStream, ReaderSettings), outputPath); xmlFileStream.Position = 0; Console.WriteLine("Converting Ways..."); ImportWays(XmlReader.Create(xmlFileStream, ReaderSettings), nodeIdRegionDict, outputPath); xmlFileStream.Dispose(); Console.WriteLine("Combining tmpFiles into region"); CombineTmpFiles(outputPath, nodeIdRegionDict.Values.ToHashSet()); } private static Dictionary GetNodesAndRegions(XmlReader xmlReader, string outputPath) { Dictionary nodeRegions = new(); Dictionary regionFileStreams = new(); Dictionary tmpAllNodes = new(); bool isHighway; HashSet currentIds = new(); while (xmlReader.Read()) { if (xmlReader.Name == "node") { ulong id = Convert.ToUInt64(xmlReader.GetAttribute("id")!); float lat = Convert.ToSingle(xmlReader.GetAttribute("lat")!, decimalInfo); float lon = Convert.ToSingle(xmlReader.GetAttribute("lon")!, decimalInfo); tmpAllNodes.TryAdd(id, new OsmNode(id, lat, lon)); } else if (xmlReader.Name == "way") { isHighway = false; currentIds.Clear(); XmlReader wayReader = xmlReader.ReadSubtree(); while (wayReader.Read()) { if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway")) { isHighway = true; } else if (xmlReader.Name == "nd") { ulong id = Convert.ToUInt64(xmlReader.GetAttribute("ref")!); currentIds.Add(id); } } wayReader.Close(); foreach (ulong nodeId in currentIds.Where(key => tmpAllNodes.ContainsKey(key))) { BinaryFormatter bFormatter = new BinaryFormatter(); if (isHighway) { ulong regionHash = Coordinates.GetRegionHashCode(tmpAllNodes[nodeId].coordinates); FileStream nodesRegionStream; if(regionFileStreams.ContainsKey(regionHash)) nodesRegionStream = regionFileStreams[regionHash]; else { string regionPath = Path.Combine(outputPath, regionHash.ToString()); Directory.CreateDirectory(regionPath); string nodesRegionPath = Path.Combine(regionPath, NodesFileName); nodesRegionStream = new FileStream(nodesRegionPath, FileMode.Create); regionFileStreams.Add(regionHash, nodesRegionStream); } nodeRegions.Add(nodeId, regionHash); #pragma warning disable SYSLIB0011 //eheheh bFormatter.Serialize(nodesRegionStream, tmpAllNodes[nodeId]); #pragma warning restore SYSLIB0011 } tmpAllNodes.Remove(nodeId); } } } xmlReader.Close(); return nodeRegions; } private static void ImportWays(XmlReader xmlReader, Dictionary nodeRegions, string outputPath) { List currentNodeIds = new(); Dictionary currentTags = new(); Dictionary regionWaysFileStreams = new(); Dictionary regionTagsFileStreams = new(); Dictionary> writtenWayTagsInRegion = new(); while (xmlReader.ReadToFollowing("way")) { currentNodeIds.Clear(); currentTags.Clear(); XmlReader wayReader = xmlReader.ReadSubtree(); while (wayReader.Read()) { currentTags.TryAdd(Tag.TagType.id, Convert.ToUInt64(wayReader.GetAttribute("id")!)); if (wayReader.Name == "tag") { HashSet pTags = Tag.ConvertToTags(wayReader.GetAttribute("k")!, wayReader.GetAttribute("v")!); if(pTags.Count > 0) foreach (Tag pTag in pTags) currentTags.TryAdd(pTag.key, pTag.value); } else if (wayReader.Name == "nd") { ulong nodeId = Convert.ToUInt64(wayReader.GetAttribute("ref")); currentNodeIds.Add(nodeId); } } wayReader.Close(); if (currentTags.ContainsKey(Tag.TagType.highway)) { for (int i = 0; i < currentNodeIds.Count - 1; i++) { ulong node1Id = currentNodeIds[i]; ulong node2Id = currentNodeIds[i+1]; if (nodeRegions.ContainsKey(node1Id) && nodeRegions.ContainsKey(node2Id)) { if (currentTags.ContainsKey(Tag.TagType.oneway) && (bool)currentTags[Tag.TagType.oneway]) { if (currentTags.ContainsKey(Tag.TagType.forward) && !(bool)currentTags[Tag.TagType.forward]) { OsmEdge n21e = new OsmEdge(currentTags[Tag.TagType.id], node2Id, node1Id, nodeRegions[node1Id]); WriteWay(ref regionWaysFileStreams, nodeRegions[node2Id], n21e, outputPath); WriteTags(ref regionTagsFileStreams, ref writtenWayTagsInRegion, nodeRegions[node2Id], currentTags, outputPath); } else { OsmEdge n12e = new OsmEdge(currentTags[Tag.TagType.id], node1Id, node2Id, nodeRegions[node2Id]); WriteWay(ref regionWaysFileStreams, nodeRegions[node1Id], n12e, outputPath); WriteTags(ref regionTagsFileStreams, ref writtenWayTagsInRegion, nodeRegions[node1Id], currentTags, outputPath); } } else { OsmEdge n12e = new OsmEdge(currentTags[Tag.TagType.id], node1Id, node2Id, nodeRegions[node2Id]); WriteWay(ref regionWaysFileStreams, nodeRegions[node1Id], n12e, outputPath); WriteTags(ref regionTagsFileStreams, ref writtenWayTagsInRegion, nodeRegions[node1Id], currentTags, outputPath); OsmEdge n21e = new OsmEdge(currentTags[Tag.TagType.id], node2Id, node1Id, nodeRegions[node1Id]); WriteWay(ref regionWaysFileStreams, nodeRegions[node2Id], n21e, outputPath); WriteTags(ref regionTagsFileStreams, ref writtenWayTagsInRegion, nodeRegions[node2Id], currentTags, outputPath); } } } } } xmlReader.Close(); foreach (FileStream f in regionWaysFileStreams.Values) f.Dispose(); foreach(FileStream f in regionTagsFileStreams.Values) f.Dispose(); } private static void WriteWay(ref Dictionary regionWaysFileStreams, ulong regionHash, OsmEdge edge, string outputPath) { BinaryFormatter bFormatter = new BinaryFormatter(); if (!regionWaysFileStreams.ContainsKey(regionHash)) { string waysRegionPath = Path.Combine(outputPath, regionHash.ToString(), WaysFileName); regionWaysFileStreams.Add(regionHash, new FileStream(waysRegionPath, FileMode.OpenOrCreate)); } #pragma warning disable SYSLIB0011 bFormatter.Serialize(regionWaysFileStreams[regionHash], edge); #pragma warning restore SYSLIB0011 } private static void WriteTags(ref Dictionary regionTagsFileStreams, ref Dictionary> writtenWayTagsInRegion, ulong regionHash, Dictionary currentTags, string outputPath) { if (writtenWayTagsInRegion.ContainsKey(regionHash) && writtenWayTagsInRegion[regionHash].Contains(currentTags[Tag.TagType.id])) return; else if(!writtenWayTagsInRegion.ContainsKey(regionHash)) writtenWayTagsInRegion.Add(regionHash, new HashSet()); BinaryFormatter bFormatter = new BinaryFormatter(); if (!regionTagsFileStreams.ContainsKey(regionHash)) { string tagsRegionPath = Path.Combine(outputPath, regionHash.ToString(), TagsFileName); regionTagsFileStreams.Add(regionHash, new FileStream(tagsRegionPath, FileMode.OpenOrCreate)); } ulong wayId = currentTags[Tag.TagType.id]; writtenWayTagsInRegion[regionHash].Add(wayId); TagManager tm = new TagManager(); foreach(KeyValuePair kv in currentTags) tm.AddTag(wayId, kv); #pragma warning disable SYSLIB0011 bFormatter.Serialize(regionTagsFileStreams[regionHash], tm); #pragma warning restore SYSLIB0011 } private static void CombineTmpFiles(string folderPath, HashSet regionHashes) { foreach (ulong regionId in regionHashes) { ValueTuple> tmpRegion = LoadRegion(folderPath, regionId); foreach (OsmEdge edge in tmpRegion.Item2) { OsmNode? startNode = tmpRegion.Item1.GetNode(edge.startId); if (startNode is not null) { startNode.edges.Add(edge); } } using (FileStream tmpRegionFileStream = new FileStream(Path.Join(folderPath, $"{regionId}{RegionsFileName}"), FileMode.Create)) { JsonSerializer.Serialize(tmpRegionFileStream, tmpRegion.Item1, typeof(Region), Region.serializerOptions); } Directory.Delete(Path.Join(folderPath, regionId.ToString()), true); } } private static ValueTuple> LoadRegion(string folderPath, ulong regionHash) { Region newRegion = new Region(regionHash); HashSet ways = new(); BinaryFormatter bFormatter = new BinaryFormatter(); if (!Directory.Exists(Path.Join(folderPath, regionHash.ToString()))) throw new FileNotFoundException("Region does not exist"); #pragma warning disable SYSLIB0011 string waysPath = Path.Join(folderPath, regionHash.ToString(), RegionConverter.WaysFileName); if(File.Exists(waysPath)) using (FileStream wayFileStream = new FileStream(waysPath, FileMode.Open)) { while (wayFileStream.Position < wayFileStream.Length) { OsmEdge deserializedEdge = (OsmEdge)bFormatter.Deserialize(wayFileStream); ways.Add(deserializedEdge); } } using (FileStream nodeFileStream = new FileStream(Path.Join(folderPath, regionHash.ToString(), RegionConverter.NodesFileName), FileMode.Open)) { while (nodeFileStream.Position < nodeFileStream.Length) { OsmNode deserializedNode = (OsmNode)bFormatter.Deserialize(nodeFileStream); newRegion.nodes.Add(deserializedNode); } } string tagsPath = Path.Join(folderPath, regionHash.ToString(), RegionConverter.TagsFileName); if(File.Exists(tagsPath)) using (FileStream tagsFileStream = new FileStream(tagsPath, FileMode.Open)) { while (tagsFileStream.Position < tagsFileStream.Length) { TagManager tm = (TagManager)bFormatter.Deserialize(tagsFileStream); ulong id = (ulong)tm.wayTagSets.First()!.Value.First(tag => tag.key == Tag.TagType.id)!.value; foreach(Tag tag in tm.wayTagSets.First()!.Value) newRegion.tagManager.AddTag(id, tag); } } #pragma warning restore SYSLIB0011 return new ValueTuple>(newRegion, ways); } }