From 6ef24eb30cc4d4c8c97e5ef8f306bc27a7fd815a Mon Sep 17 00:00:00 2001 From: glax Date: Mon, 22 Jul 2024 04:57:03 +0200 Subject: [PATCH] New Region file layout --- .gitignore | 5 + .idea/.idea.OSM_Regions/.idea/.gitignore | 13 + .idea/.idea.OSM_Regions/.idea/encodings.xml | 4 + .idea/.idea.OSM_Regions/.idea/indexLayout.xml | 8 + .idea/.idea.OSM_Regions/.idea/vcs.xml | 6 + OSM_Regions.sln | 16 ++ OSM_Regions.sln.DotSettings.user | 4 + OSM_Regions/Converter.cs | 265 ++++++++++++++++++ OSM_Regions/OSM_Regions.csproj | 25 ++ OSM_Regions/Program.cs | 38 +++ OSM_Regions/RegionLoader.cs | 134 +++++++++ OSM_Regions/Utils/RegionUtils.cs | 18 ++ 12 files changed, 536 insertions(+) create mode 100644 .gitignore create mode 100644 .idea/.idea.OSM_Regions/.idea/.gitignore create mode 100644 .idea/.idea.OSM_Regions/.idea/encodings.xml create mode 100644 .idea/.idea.OSM_Regions/.idea/indexLayout.xml create mode 100644 .idea/.idea.OSM_Regions/.idea/vcs.xml create mode 100644 OSM_Regions.sln create mode 100644 OSM_Regions.sln.DotSettings.user create mode 100644 OSM_Regions/Converter.cs create mode 100644 OSM_Regions/OSM_Regions.csproj create mode 100644 OSM_Regions/Program.cs create mode 100644 OSM_Regions/RegionLoader.cs create mode 100644 OSM_Regions/Utils/RegionUtils.cs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..add57be --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +bin/ +obj/ +/packages/ +riderModule.iml +/_ReSharper.Caches/ \ No newline at end of file diff --git a/.idea/.idea.OSM_Regions/.idea/.gitignore b/.idea/.idea.OSM_Regions/.idea/.gitignore new file mode 100644 index 0000000..154e291 --- /dev/null +++ b/.idea/.idea.OSM_Regions/.idea/.gitignore @@ -0,0 +1,13 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Rider ignored files +/contentModel.xml +/.idea.OSM_Regions.iml +/modules.xml +/projectSettingsUpdater.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/.idea.OSM_Regions/.idea/encodings.xml b/.idea/.idea.OSM_Regions/.idea/encodings.xml new file mode 100644 index 0000000..df87cf9 --- /dev/null +++ b/.idea/.idea.OSM_Regions/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/.idea.OSM_Regions/.idea/indexLayout.xml b/.idea/.idea.OSM_Regions/.idea/indexLayout.xml new file mode 100644 index 0000000..7b08163 --- /dev/null +++ b/.idea/.idea.OSM_Regions/.idea/indexLayout.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/.idea.OSM_Regions/.idea/vcs.xml b/.idea/.idea.OSM_Regions/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/.idea.OSM_Regions/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/OSM_Regions.sln b/OSM_Regions.sln new file mode 100644 index 0000000..4ce90b5 --- /dev/null +++ b/OSM_Regions.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OSM_Regions", "OSM_Regions\OSM_Regions.csproj", "{CD362E97-AA6D-446E-8B72-102913BFDC62}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CD362E97-AA6D-446E-8B72-102913BFDC62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CD362E97-AA6D-446E-8B72-102913BFDC62}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CD362E97-AA6D-446E-8B72-102913BFDC62}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CD362E97-AA6D-446E-8B72-102913BFDC62}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/OSM_Regions.sln.DotSettings.user b/OSM_Regions.sln.DotSettings.user new file mode 100644 index 0000000..3c88bc7 --- /dev/null +++ b/OSM_Regions.sln.DotSettings.user @@ -0,0 +1,4 @@ + + True + True + True \ No newline at end of file diff --git a/OSM_Regions/Converter.cs b/OSM_Regions/Converter.cs new file mode 100644 index 0000000..aad5a4b --- /dev/null +++ b/OSM_Regions/Converter.cs @@ -0,0 +1,265 @@ +using System.Globalization; +using System.Runtime.Serialization; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; +using Microsoft.Extensions.Logging; +using OSM_Graph; + +namespace OSM_Regions; + +public class Converter(float regionSize, string? exportFolderPath = null, ILogger? logger = null) +{ + internal readonly float RegionSize = regionSize; + + internal readonly string ExportFolderPath = exportFolderPath ?? Path.Join(Environment.CurrentDirectory, regionSize.ToString(CultureInfo.InvariantCulture)); + internal const string NodesMapRegionFileName = "nodes.map"; + internal const string WayMapRegionFileName = "ways.map"; + internal const string NodesRegionDirectory = "nodes"; + internal const string WaysRegionDirectory = "ways"; + + private static readonly FileStreamOptions DefaultWriteOptions = new () + { + Access = FileAccess.Write, + Mode = FileMode.OpenOrCreate, + Share = FileShare.Read + }; + + private static readonly FileStreamOptions DefaultReadOptions = new () + { + Access = FileAccess.Read, + Mode = FileMode.Open, + Share = FileShare.Read + }; + + private static readonly XmlReaderSettings ReaderSettings = new() + { + IgnoreWhitespace = true, + IgnoreComments = true + }; + + public void SplitOsmExportIntoRegionFiles(string osmExportFilePath) + { + logger?.LogInformation($"Output Path {ExportFolderPath}"); + Directory.CreateDirectory(ExportFolderPath); + + logger?.LogInformation($"Opening OSM-Export-File... ({osmExportFilePath})"); + FileStream osmExportFileStream = new (osmExportFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); + + logger?.LogInformation("Creating Map-Files..."); + StreamWriter nodesMapFileStream = new(Path.Join(ExportFolderPath, NodesMapRegionFileName), Encoding.ASCII, DefaultWriteOptions); + Dictionary nodesRegionStreamWriters = new(); + StreamWriter waysMapFileStream = new(Path.Join(ExportFolderPath, WayMapRegionFileName), Encoding.ASCII, DefaultWriteOptions); + Dictionary waysRegionStreamWriters = new(); + + logger?.LogInformation("Splitting Nodes..."); + Dictionary nodeRegionIdMap = new(); + + DateTime start = DateTime.Now;; + DateTime print = DateTime.Now; + XmlReader reader = XmlReader.Create(osmExportFileStream, ReaderSettings); + reader.MoveToContent(); + while (reader.ReadToFollowing("node")) + { + if (DateTime.Now - print > TimeSpan.FromSeconds(2)) + { + float finished = osmExportFileStream.Position * 1f / osmExportFileStream.Length; + TimeSpan elapsed = DateTime.Now - start; + TimeSpan remaining = elapsed / finished * (1 - finished); + logger?.LogDebug($"{finished:P} {elapsed:hh\\:mm\\:ss} elapsed {remaining:hh\\:mm\\:ss} remaining ({osmExportFileStream.Position}/{osmExportFileStream.Length})"); + print = DateTime.Now; + } + try + { + ulong id = ulong.Parse(reader.GetAttribute("id") ?? string.Empty); + float lat = float.Parse(reader.GetAttribute("lat") ?? string.Empty, NumberStyles.Float, CultureInfo.InvariantCulture); + float lon = float.Parse(reader.GetAttribute("lon") ?? string.Empty, NumberStyles.Float, CultureInfo.InvariantCulture); + Node n = new (id, lat, lon); + + long regionFileStreamId = Utils.RegionUtils.GetRegionIdFromCoordinates(lat, lon, regionSize); + StreamWriter sr = GetOrCreateRegionStreamWriter(regionFileStreamId, ref nodesRegionStreamWriters, RegionType.Node); + string line = n.Serialize(); + sr.WriteLine(line); + logger?.LogTrace($"{regionFileStreamId} -> {line}"); + + //nodeId-{regionId}\n + nodesMapFileStream.WriteLine($"{id}-{regionFileStreamId}"); + nodeRegionIdMap.Add(id, regionFileStreamId); + } + catch (Exception e) + { + logger?.LogError(e, $"Error parsing Node-line.\n{e}"); + } + } + logger?.LogInformation("Closing Streams..."); + foreach ((long _, StreamWriter? streamWriter) in nodesRegionStreamWriters) + streamWriter.Dispose(); + nodesMapFileStream.Dispose(); + + logger?.LogInformation("Splitting Ways..."); + osmExportFileStream.Position = 0; + reader = XmlReader.Create(osmExportFileStream, ReaderSettings); + reader.MoveToContent(); + + while (reader.ReadToFollowing("way")) + { + if (DateTime.Now - print > TimeSpan.FromSeconds(2)) + { + TimeSpan elapsed = DateTime.Now - start; + float finished = osmExportFileStream.Position * 1f / osmExportFileStream.Length; + logger?.LogDebug($"{finished:P} {elapsed:hh\\:mm\\:ss} elapsed {elapsed/finished*(1-finished):hh\\:mm\\:ss} remaining ({osmExportFileStream.Position}/{osmExportFileStream.Length})"); + print = DateTime.Now; + } + try + { + ulong id = ulong.Parse(reader.GetAttribute("id") ?? string.Empty); + List nodeIds = new(); + Dictionary tags = new(); + using (XmlReader wayReader = reader.ReadSubtree()) + { + while (wayReader.Read()) + { + if (reader.Name == "tag") + { + string? key = reader.GetAttribute("k"); + string? value = reader.GetAttribute("v"); + if(value is null || key is null) + continue; + tags.Add(key, value); + } + else if (reader.Name == "nd") + { + string? nodeId = reader.GetAttribute("ref"); + if(nodeId is null) + continue; + nodeIds.Add(ulong.Parse(nodeId)); + } + } + } + + List regionIds = nodeIds.Select(nId => nodeRegionIdMap[nId]).Distinct().ToList(); + Way way = new (id, tags, nodeIds); + foreach (long regionId in regionIds) + { + StreamWriter sr = GetOrCreateRegionStreamWriter(regionId, ref waysRegionStreamWriters, RegionType.Way); + string line = way.Serialize(); + sr.WriteLine(line); + logger?.LogTrace($"{regionId} -> {line}"); + } + waysMapFileStream.WriteLine($"{id}-{string.Join(',', regionIds)}"); + } + catch (Exception e) + { + logger?.LogError(e, $"Error parsing Way-line.\n{e}"); + } + } + logger?.LogInformation("Closing Streams..."); + foreach ((long _, StreamWriter? streamWriter) in waysRegionStreamWriters) + streamWriter.Dispose(); + waysMapFileStream.Dispose(); + logger?.LogInformation("Removing Non-Highways."); + RemoveAllNonHighways(); + logger?.LogInformation("Done!"); + } + + public void RemoveAllNonHighways() + { + string[] wayRegionsPaths = Directory.GetFiles(Path.Join(ExportFolderPath, WaysRegionDirectory)).Where(file => !Regex.IsMatch(file, @"[0-9]+\..*")).ToArray(); + int count = 0; + DateTime print = DateTime.Now; + DateTime start = DateTime.Now; + foreach (string path in wayRegionsPaths) + { + if (DateTime.Now - print > TimeSpan.FromSeconds(2)) + { + float finished = count * 1f / wayRegionsPaths.Length; + TimeSpan elapsed = DateTime.Now - start; + TimeSpan remaining = elapsed / finished * (1 - finished); + logger?.LogDebug($"{finished:P} {elapsed:hh\\:mm\\:ss} elapsed {remaining:hh\\:mm\\:ss} remaining ({count}/{wayRegionsPaths.Length})"); + print = DateTime.Now; + } + count++; + + List nodeIds = new(); + File.Copy(path, $"{path}.bak", true); + StreamReader waysStreamReader = new(path, Encoding.UTF8, false, DefaultReadOptions); + StreamWriter waysStreamWriter = new($"{path}.new", Encoding.UTF8, DefaultWriteOptions); + bool hasWritten = false; + while (!waysStreamReader.EndOfStream) + { + string? line = waysStreamReader.ReadLine(); + if (line is null) + continue; + try + { + Way w = Way.Deserialize(line); + if (w.Tags.ContainsKey("highway")) + { + waysStreamWriter.WriteLine(line); + nodeIds.AddRange(w.NodeIds); + hasWritten = true; + }else + logger?.LogTrace($"Way {w.ID} is not a highway. BYE!"); + } + catch (SerializationException e) + { + logger?.LogWarning(e.Message); + } + } + string nodesPath = Path.Join(ExportFolderPath, NodesRegionDirectory, new FileInfo(path).Name); + + waysStreamReader.Dispose(); + waysStreamWriter.Dispose(); + + if (!hasWritten) + { + logger?.LogTrace($"Region {path} had no Highways. BYE!"); + File.Delete($"{path}.new"); + File.Delete(path); + File.Move(nodesPath, $"{nodesPath}.bak", true); + } + else + { + File.Move($"{path}.new", path, true); + + File.Copy(nodesPath, $"{nodesPath}.bak", true); + StreamReader nodesStreamReader = new(nodesPath, Encoding.UTF8, false, DefaultReadOptions); + StreamWriter nodesStreamWriter = new($"{nodesPath}.new", Encoding.UTF8, DefaultWriteOptions); + while (!nodesStreamReader.EndOfStream) + { + string? line = nodesStreamReader.ReadLine(); + if(line is null) + continue; + Node n = Node.Deserialize(line); + if(nodeIds.Contains(n.ID)) + nodesStreamWriter.WriteLine(line); + } + nodesStreamReader.Dispose(); + nodesStreamWriter.Dispose(); + File.Move($"{nodesPath}.new", nodesPath, true); + } + } + + logger?.LogInformation("Removing .bak files..."); + foreach (string bakFile in Directory.GetFiles(Path.Join(ExportFolderPath, WaysRegionDirectory), "*.bak") + .Concat(Directory.GetFiles(Path.Join(ExportFolderPath, NodesRegionDirectory), "*.bak"))) + File.Delete(bakFile); + } + + private StreamWriter GetOrCreateRegionStreamWriter(long regionId, ref Dictionary srDict, RegionType regionType) + { + string directory = regionType == RegionType.Node ? NodesRegionDirectory : WaysRegionDirectory; + + if (!srDict.TryGetValue(regionId, out StreamWriter? ret)) + { + string filePath = Path.Join(ExportFolderPath, directory, regionId.ToString()); + logger?.LogTrace($"Creating FileStream {filePath} for Region {regionId}"); + Directory.CreateDirectory(Path.Join(ExportFolderPath, directory)); + ret = new StreamWriter(filePath, Encoding.UTF8, DefaultWriteOptions); + srDict.Add(regionId, ret); + } + return ret; + } + + private enum RegionType : byte { Node, Way} +} \ No newline at end of file diff --git a/OSM_Regions/OSM_Regions.csproj b/OSM_Regions/OSM_Regions.csproj new file mode 100644 index 0000000..ac1b602 --- /dev/null +++ b/OSM_Regions/OSM_Regions.csproj @@ -0,0 +1,25 @@ + + + + Exe + net8.0 + enable + enable + + + + + ..\..\OSM_Graph\OSM_Graph\bin\Debug\net8.0\Graph.dll + + + ..\..\OSM_Graph\OSM_Graph\bin\Debug\net8.0\OSM_Graph.dll + + + + + + + + + + diff --git a/OSM_Regions/Program.cs b/OSM_Regions/Program.cs new file mode 100644 index 0000000..8db168e --- /dev/null +++ b/OSM_Regions/Program.cs @@ -0,0 +1,38 @@ +using System.Globalization; +using GlaxArguments; +using GlaxLogger; +using Microsoft.Extensions.Logging; +using OSM_Regions; + +Logger logger = new(LogLevel.Trace, consoleOut: Console.Out); + +Argument regionSizeArgument = new (["-r", "--regionSize"], 1, "Size of Regions (f.e. 0.001)"); +Argument convertArgument = new (["-c", "--convert"], 1, "Converts a OSM-XML-Export to Region-files."); +Argument exportPathArgument = new (["-e", "--exportPath"], 1, "Export Directory Path."); + +ArgumentFetcher af = new ([regionSizeArgument, convertArgument]); +Dictionary arguments = af.Fetch(args); + +if(!arguments.TryGetValue(regionSizeArgument, out string[]? regionSizeVars) || !arguments.TryGetValue(convertArgument, out string[]? osmFilePathVars)) + PrintUsage(); + +float regionSize; +try +{ + regionSize = float.Parse(regionSizeVars?[0] ?? string.Empty, NumberStyles.Float, NumberFormatInfo.InvariantInfo); +} +catch (Exception e) +{ + Console.WriteLine($"Error parsing RegionSize from input {regionSizeVars?[0]}"); + return -1; +} + +Converter converter = new (regionSize, arguments.TryGetValue(exportPathArgument, out string[]? argument) ? argument[0] : null, logger); +converter.SplitOsmExportIntoRegionFiles(arguments[convertArgument][0]); + +return 0; + +void PrintUsage() +{ + Console.WriteLine($"Usage: {AppDomain.CurrentDomain.FriendlyName} -c -r [-e ]"); +} \ No newline at end of file diff --git a/OSM_Regions/RegionLoader.cs b/OSM_Regions/RegionLoader.cs new file mode 100644 index 0000000..7266419 --- /dev/null +++ b/OSM_Regions/RegionLoader.cs @@ -0,0 +1,134 @@ +using System.Globalization; +using System.Text; +using Microsoft.Extensions.Logging; +using OSM_Graph; + +namespace OSM_Regions; + +public class RegionLoader(float regionSize, string? importFolderPath = null, ILogger? logger = null) +{ + internal readonly float RegionSize = regionSize; + + internal readonly string ImportFolderPath = importFolderPath ?? Path.Join(Environment.CurrentDirectory, regionSize.ToString(CultureInfo.InvariantCulture)); + private const string NodesMapRegionFileName = "nodes.map"; + private const string WayMapRegionFileName = "ways.map"; + + public Graph.Graph? LoadRegionFromRegionId(long regionId) + { + logger?.LogDebug($"Loading Region {regionId}"); + Graph.Graph ret = new(); + StreamReader nodesReader, waysReader; + string nodesPath = Path.Join(ImportFolderPath, Converter.NodesRegionDirectory, regionId.ToString()); + string waysPath = Path.Join(ImportFolderPath, Converter.WaysRegionDirectory, regionId.ToString()); + try + { + nodesReader = new(nodesPath, Encoding.UTF8); + } + catch (Exception e) + { + logger?.LogError(e, $"Could not find Region in Nodes-Directory. ({nodesPath})"); + return null; + } + try + { + waysReader = new(waysPath, Encoding.UTF8); + } + catch (Exception e) + { + logger?.LogError(e, $"Could not find Region in Ways-Directory. ({waysPath})"); + return null; + } + + logger?.LogDebug($"Loading Nodes {nodesPath}"); + while (!nodesReader.EndOfStream) + { + string? line = nodesReader.ReadLine(); + if(line is null) + continue; + Node n = Node.Deserialize(line); + ret.Nodes.Add(n.ID, new Graph.Node(n.Lat, n.Lon)); + } + + logger?.LogDebug($"Loading Ways {waysPath}"); + while (!waysReader.EndOfStream) + { + string? line = waysReader.ReadLine(); + if(line is null) + continue; + Way w = Way.Deserialize(line); + ret.Ways.Add(w.ID, new Graph.Way(w.Tags)); + for (int i = 1; i < w.NodeIds.Count; i++) + { + ulong node1Id = w.NodeIds[i - 1]; + ulong node2Id = w.NodeIds[i]; + if (ret.Nodes.TryGetValue(node1Id, out Graph.Node? node1) && ret.Nodes.TryGetValue(node2Id, out Graph.Node? node2)) + { + node1.Neighbors.TryAdd(node2Id, w.ID); + node2.Neighbors.TryAdd(node1Id, w.ID); + //TODO add oneway checks + } + } + } + + return ret; + } + + public Graph.Graph? LoadRegionFromNodeId(ulong nodeId) + { + logger?.LogDebug($"Loading Region for Node {nodeId}"); + StreamReader nodesMapFileStream = new(Path.Join(ImportFolderPath, NodesMapRegionFileName), Encoding.ASCII); + while (!nodesMapFileStream.EndOfStream) + { + string? line = nodesMapFileStream.ReadLine(); + if(line is null) + continue; + try + { + ulong id = ulong.Parse(line.Split('-')[0]); + if (id == nodeId) + return LoadRegionFromRegionId(long.Parse(line.Split('-')[1])); + } + catch (Exception e) + { + logger?.LogError(e, "Error parsing Node-line."); + } + } + + logger?.LogWarning($"Could not find Node {nodeId}"); + return null; + } + + public Graph.Graph?[] LoadRegionsFromWayId(ulong wayId) + { + logger?.LogDebug($"Loading Region for Way {wayId}"); + StreamReader waysMapFileStream = new(Path.Join(ImportFolderPath, WayMapRegionFileName), Encoding.ASCII); + while (!waysMapFileStream.EndOfStream) + { + string? line = waysMapFileStream.ReadLine(); + if(line is null) + continue; + try + { + ulong id = ulong.Parse(line.Split('-')[0]); + if (id == wayId) + return line.Split('-')[1] + .Split(',') + .Select(rId => LoadRegionFromRegionId(long.Parse(rId))) + .ToArray(); + } + catch (Exception e) + { + logger?.LogError(e, "Error parsing Way-line."); + } + } + + logger?.LogWarning($"Could not find Way {wayId}"); + return []; + } + + public Graph.Graph? LoadRegionFromCoordinates(float lat, float lon) + { + logger?.LogDebug($"Loading Region for Coordinates {lat} {lon}"); + return LoadRegionFromRegionId(Utils.RegionUtils.GetRegionIdFromCoordinates(lat, lon, regionSize)); + } +} \ No newline at end of file diff --git a/OSM_Regions/Utils/RegionUtils.cs b/OSM_Regions/Utils/RegionUtils.cs new file mode 100644 index 0000000..a045754 --- /dev/null +++ b/OSM_Regions/Utils/RegionUtils.cs @@ -0,0 +1,18 @@ +using System.Globalization; + +namespace OSM_Regions.Utils; + +public static class RegionUtils +{ + public static long GetRegionIdFromCoordinates(float lat, float lon, float regionSize) + { + string latStr = $"{Math.Floor(lat / regionSize):000000}".Replace(".","").Replace(",", ""); + string lonStr = $"{Math.Floor(lon / regionSize):000000}".Replace(".","").Replace(",", ""); + return long.Parse(string.Concat(latStr, lonStr)); + } + + public static long GetRegionIdFromCoordinates(string lat, string lon, float regionSize) => + GetRegionIdFromCoordinates(float.Parse(lat, NumberStyles.Float, NumberFormatInfo.InvariantInfo), + float.Parse(lon, NumberStyles.Float, NumberFormatInfo.InvariantInfo), + regionSize); +} \ No newline at end of file