glax/OSM_Regions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New Region file layout

Browse Source
This commit is contained in:
glax 2024-07-22 04:57:03 +02:00
commit 6ef24eb30c
12 changed files with 536 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
bin/
obj/
/packages/
riderModule.iml
/_ReSharper.Caches/

13
.idea/.idea.OSM_Regions/.idea/.gitignore vendored Normal file
View File

@ -0,0 +1,13 @@
# Default ignored files
/shelf/
/workspace.xml
# Rider ignored files
/contentModel.xml
/.idea.OSM_Regions.iml
/modules.xml
/projectSettingsUpdater.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

4
.idea/.idea.OSM_Regions/.idea/encodings.xml Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" addBOMForNewFiles="with BOM under Windows, with no BOM otherwise" />
</project>

8
.idea/.idea.OSM_Regions/.idea/indexLayout.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="UserContentModel">
<attachedFolders />
<explicitIncludes />
<explicitExcludes />
</component>
</project>

6
.idea/.idea.OSM_Regions/.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

16
OSM_Regions.sln Normal file
View File

@ -0,0 +1,16 @@

Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OSM_Regions", "OSM_Regions\OSM_Regions.csproj", "{CD362E97-AA6D-446E-8B72-102913BFDC62}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{CD362E97-AA6D-446E-8B72-102913BFDC62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CD362E97-AA6D-446E-8B72-102913BFDC62}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CD362E97-AA6D-446E-8B72-102913BFDC62}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CD362E97-AA6D-446E-8B72-102913BFDC62}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

4
OSM_Regions.sln.DotSettings.user Normal file
View File

@ -0,0 +1,4 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:Boolean x:Key="/Default/AddReferences/RecentPaths/=C_003A_005CUsers_005CGlax_005CRiderProjects_005COSM_005FGraph_005COSM_005FGraph_005Cbin_005CDebug_005Cnet8_002E0_005CGraph_002Edll/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/AddReferences/RecentPaths/=C_003A_005CUsers_005CGlax_005CRiderProjects_005COSM_005FGraph_005COSM_005FGraph_005Cbin_005CDebug_005Cnet8_002E0_005COSM_005FGraph_002Edll/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/AddReferences/RecentPaths/=C_003A_005CUsers_005CGlax_005CRiderProjects_005COSM_005FGraph_005COSM_005FGraph_005Cbin_005CDebug_005Cnet8_002E0_005COSM_005FGraph_002Eexe/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

265
OSM_Regions/Converter.cs Normal file
View File

@ -0,0 +1,265 @@
using System.Globalization;
using System.Runtime.Serialization;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using Microsoft.Extensions.Logging;
using OSM_Graph;
namespace OSM_Regions;
public class Converter(float regionSize, string? exportFolderPath = null, ILogger? logger = null)
{
internal readonly float RegionSize = regionSize;
internal readonly string ExportFolderPath = exportFolderPath ?? Path.Join(Environment.CurrentDirectory, regionSize.ToString(CultureInfo.InvariantCulture));
internal const string NodesMapRegionFileName = "nodes.map";
internal const string WayMapRegionFileName = "ways.map";
internal const string NodesRegionDirectory = "nodes";
internal const string WaysRegionDirectory = "ways";
private static readonly FileStreamOptions DefaultWriteOptions = new ()
{
Access = FileAccess.Write,
Mode = FileMode.OpenOrCreate,
Share = FileShare.Read
};
private static readonly FileStreamOptions DefaultReadOptions = new ()
{
Access = FileAccess.Read,
Mode = FileMode.Open,
Share = FileShare.Read
};
private static readonly XmlReaderSettings ReaderSettings = new()
{
IgnoreWhitespace = true,
IgnoreComments = true
};
public void SplitOsmExportIntoRegionFiles(string osmExportFilePath)
{
logger?.LogInformation($"Output Path {ExportFolderPath}");
Directory.CreateDirectory(ExportFolderPath);
logger?.LogInformation($"Opening OSM-Export-File... ({osmExportFilePath})");
FileStream osmExportFileStream = new (osmExportFilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
logger?.LogInformation("Creating Map-Files...");
StreamWriter nodesMapFileStream = new(Path.Join(ExportFolderPath, NodesMapRegionFileName), Encoding.ASCII, DefaultWriteOptions);
Dictionary<long, StreamWriter> nodesRegionStreamWriters = new();
StreamWriter waysMapFileStream = new(Path.Join(ExportFolderPath, WayMapRegionFileName), Encoding.ASCII, DefaultWriteOptions);
Dictionary<long, StreamWriter> waysRegionStreamWriters = new();
logger?.LogInformation("Splitting Nodes...");
Dictionary<ulong, long> nodeRegionIdMap = new();
DateTime start = DateTime.Now;;
DateTime print = DateTime.Now;
XmlReader reader = XmlReader.Create(osmExportFileStream, ReaderSettings);
reader.MoveToContent();
while (reader.ReadToFollowing("node"))
{
if (DateTime.Now - print > TimeSpan.FromSeconds(2))
{
float finished = osmExportFileStream.Position * 1f / osmExportFileStream.Length;
TimeSpan elapsed = DateTime.Now - start;
TimeSpan remaining = elapsed / finished * (1 - finished);
logger?.LogDebug($"{finished:P} {elapsed:hh\\:mm\\:ss} elapsed {remaining:hh\\:mm\\:ss} remaining ({osmExportFileStream.Position}/{osmExportFileStream.Length})");
print = DateTime.Now;
}
try
{
ulong id = ulong.Parse(reader.GetAttribute("id") ?? string.Empty);
float lat = float.Parse(reader.GetAttribute("lat") ?? string.Empty, NumberStyles.Float, CultureInfo.InvariantCulture);
float lon = float.Parse(reader.GetAttribute("lon") ?? string.Empty, NumberStyles.Float, CultureInfo.InvariantCulture);
Node n = new (id, lat, lon);
long regionFileStreamId = Utils.RegionUtils.GetRegionIdFromCoordinates(lat, lon, regionSize);
StreamWriter sr = GetOrCreateRegionStreamWriter(regionFileStreamId, ref nodesRegionStreamWriters, RegionType.Node);
string line = n.Serialize();
sr.WriteLine(line);
logger?.LogTrace($"{regionFileStreamId} -> {line}");
//nodeId-{regionId}\n
nodesMapFileStream.WriteLine($"{id}-{regionFileStreamId}");
nodeRegionIdMap.Add(id, regionFileStreamId);
}
catch (Exception e)
{
logger?.LogError(e, $"Error parsing Node-line.\n{e}");
}
}
logger?.LogInformation("Closing Streams...");
foreach ((long _, StreamWriter? streamWriter) in nodesRegionStreamWriters)
streamWriter.Dispose();
nodesMapFileStream.Dispose();
logger?.LogInformation("Splitting Ways...");
osmExportFileStream.Position = 0;
reader = XmlReader.Create(osmExportFileStream, ReaderSettings);
reader.MoveToContent();
while (reader.ReadToFollowing("way"))
{
if (DateTime.Now - print > TimeSpan.FromSeconds(2))
{
TimeSpan elapsed = DateTime.Now - start;
float finished = osmExportFileStream.Position * 1f / osmExportFileStream.Length;
logger?.LogDebug($"{finished:P} {elapsed:hh\\:mm\\:ss} elapsed {elapsed/finished*(1-finished):hh\\:mm\\:ss} remaining ({osmExportFileStream.Position}/{osmExportFileStream.Length})");
print = DateTime.Now;
}
try
{
ulong id = ulong.Parse(reader.GetAttribute("id") ?? string.Empty);
List<ulong> nodeIds = new();
Dictionary<string, string> tags = new();
using (XmlReader wayReader = reader.ReadSubtree())
{
while (wayReader.Read())
{
if (reader.Name == "tag")
{
string? key = reader.GetAttribute("k");
string? value = reader.GetAttribute("v");
if(value is null || key is null)
continue;
tags.Add(key, value);
}
else if (reader.Name == "nd")
{
string? nodeId = reader.GetAttribute("ref");
if(nodeId is null)
continue;
nodeIds.Add(ulong.Parse(nodeId));
}
}
}
List<long> regionIds = nodeIds.Select(nId => nodeRegionIdMap[nId]).Distinct().ToList();
Way way = new (id, tags, nodeIds);
foreach (long regionId in regionIds)
{
StreamWriter sr = GetOrCreateRegionStreamWriter(regionId, ref waysRegionStreamWriters, RegionType.Way);
string line = way.Serialize();
sr.WriteLine(line);
logger?.LogTrace($"{regionId} -> {line}");
}
waysMapFileStream.WriteLine($"{id}-{string.Join(',', regionIds)}");
}
catch (Exception e)
{
logger?.LogError(e, $"Error parsing Way-line.\n{e}");
}
}
logger?.LogInformation("Closing Streams...");
foreach ((long _, StreamWriter? streamWriter) in waysRegionStreamWriters)
streamWriter.Dispose();
waysMapFileStream.Dispose();
logger?.LogInformation("Removing Non-Highways.");
RemoveAllNonHighways();
logger?.LogInformation("Done!");
}
public void RemoveAllNonHighways()
{
string[] wayRegionsPaths = Directory.GetFiles(Path.Join(ExportFolderPath, WaysRegionDirectory)).Where(file => !Regex.IsMatch(file, @"[0-9]+\..*")).ToArray();
int count = 0;
DateTime print = DateTime.Now;
DateTime start = DateTime.Now;
foreach (string path in wayRegionsPaths)
{
if (DateTime.Now - print > TimeSpan.FromSeconds(2))
{
float finished = count * 1f / wayRegionsPaths.Length;
TimeSpan elapsed = DateTime.Now - start;
TimeSpan remaining = elapsed / finished * (1 - finished);
logger?.LogDebug($"{finished:P} {elapsed:hh\\:mm\\:ss} elapsed {remaining:hh\\:mm\\:ss} remaining ({count}/{wayRegionsPaths.Length})");
print = DateTime.Now;
}
count++;
List<ulong> nodeIds = new();
File.Copy(path, $"{path}.bak", true);
StreamReader waysStreamReader = new(path, Encoding.UTF8, false, DefaultReadOptions);
StreamWriter waysStreamWriter = new($"{path}.new", Encoding.UTF8, DefaultWriteOptions);
bool hasWritten = false;
while (!waysStreamReader.EndOfStream)
{
string? line = waysStreamReader.ReadLine();
if (line is null)
continue;
try
{
Way w = Way.Deserialize(line);
if (w.Tags.ContainsKey("highway"))
{
waysStreamWriter.WriteLine(line);
nodeIds.AddRange(w.NodeIds);
hasWritten = true;
}else
logger?.LogTrace($"Way {w.ID} is not a highway. BYE!");
}
catch (SerializationException e)
{
logger?.LogWarning(e.Message);
}
}
string nodesPath = Path.Join(ExportFolderPath, NodesRegionDirectory, new FileInfo(path).Name);
waysStreamReader.Dispose();
waysStreamWriter.Dispose();
if (!hasWritten)
{
logger?.LogTrace($"Region {path} had no Highways. BYE!");
File.Delete($"{path}.new");
File.Delete(path);
File.Move(nodesPath, $"{nodesPath}.bak", true);
}
else
{
File.Move($"{path}.new", path, true);
File.Copy(nodesPath, $"{nodesPath}.bak", true);
StreamReader nodesStreamReader = new(nodesPath, Encoding.UTF8, false, DefaultReadOptions);
StreamWriter nodesStreamWriter = new($"{nodesPath}.new", Encoding.UTF8, DefaultWriteOptions);
while (!nodesStreamReader.EndOfStream)
{
string? line = nodesStreamReader.ReadLine();
if(line is null)
continue;
Node n = Node.Deserialize(line);
if(nodeIds.Contains(n.ID))
nodesStreamWriter.WriteLine(line);
}
nodesStreamReader.Dispose();
nodesStreamWriter.Dispose();
File.Move($"{nodesPath}.new", nodesPath, true);
}
}
logger?.LogInformation("Removing .bak files...");
foreach (string bakFile in Directory.GetFiles(Path.Join(ExportFolderPath, WaysRegionDirectory), "*.bak")
.Concat(Directory.GetFiles(Path.Join(ExportFolderPath, NodesRegionDirectory), "*.bak")))
File.Delete(bakFile);
}
private StreamWriter GetOrCreateRegionStreamWriter(long regionId, ref Dictionary<long, StreamWriter> srDict, RegionType regionType)
{
string directory = regionType == RegionType.Node ? NodesRegionDirectory : WaysRegionDirectory;
if (!srDict.TryGetValue(regionId, out StreamWriter? ret))
{
string filePath = Path.Join(ExportFolderPath, directory, regionId.ToString());
logger?.LogTrace($"Creating FileStream {filePath} for Region {regionId}");
Directory.CreateDirectory(Path.Join(ExportFolderPath, directory));
ret = new StreamWriter(filePath, Encoding.UTF8, DefaultWriteOptions);
srDict.Add(regionId, ret);
}
return ret;
}
private enum RegionType : byte { Node, Way}
}

25
OSM_Regions/OSM_Regions.csproj Normal file
View File

@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Reference Include="Graph">
<HintPath>..\..\OSM_Graph\OSM_Graph\bin\Debug\net8.0\Graph.dll</HintPath>
</Reference>
<Reference Include="OSM_Graph">
<HintPath>..\..\OSM_Graph\OSM_Graph\bin\Debug\net8.0\OSM_Graph.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="GlaxArguments" Version="1.1.0" />
<PackageReference Include="GlaxLogger" Version="1.0.7.2" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
</ItemGroup>
</Project>

38
OSM_Regions/Program.cs Normal file
View File

@ -0,0 +1,38 @@
using System.Globalization;
using GlaxArguments;
using GlaxLogger;
using Microsoft.Extensions.Logging;
using OSM_Regions;
Logger logger = new(LogLevel.Trace, consoleOut: Console.Out);
Argument regionSizeArgument = new (["-r", "--regionSize"], 1, "Size of Regions (f.e. 0.001)");
Argument convertArgument = new (["-c", "--convert"], 1, "Converts a OSM-XML-Export to Region-files.");
Argument exportPathArgument = new (["-e", "--exportPath"], 1, "Export Directory Path.");
ArgumentFetcher af = new ([regionSizeArgument, convertArgument]);
Dictionary<Argument, string[]> arguments = af.Fetch(args);
if(!arguments.TryGetValue(regionSizeArgument, out string[]? regionSizeVars) || !arguments.TryGetValue(convertArgument, out string[]? osmFilePathVars))
PrintUsage();
float regionSize;
try
{
regionSize = float.Parse(regionSizeVars?[0] ?? string.Empty, NumberStyles.Float, NumberFormatInfo.InvariantInfo);
}
catch (Exception e)
{
Console.WriteLine($"Error parsing RegionSize from input {regionSizeVars?[0]}");
return -1;
}
Converter converter = new (regionSize, arguments.TryGetValue(exportPathArgument, out string[]? argument) ? argument[0] : null, logger);
converter.SplitOsmExportIntoRegionFiles(arguments[convertArgument][0]);
return 0;
void PrintUsage()
{
Console.WriteLine($"Usage: {AppDomain.CurrentDomain.FriendlyName} -c <Path to OSM-XML-Export-file> -r <Region Size 0.001> [-e <Export Folder Path>]");
}

134
OSM_Regions/RegionLoader.cs Normal file
View File

@ -0,0 +1,134 @@
using System.Globalization;
using System.Text;
using Microsoft.Extensions.Logging;
using OSM_Graph;
namespace OSM_Regions;
public class RegionLoader(float regionSize, string? importFolderPath = null, ILogger? logger = null)
{
internal readonly float RegionSize = regionSize;
internal readonly string ImportFolderPath = importFolderPath ?? Path.Join(Environment.CurrentDirectory, regionSize.ToString(CultureInfo.InvariantCulture));
private const string NodesMapRegionFileName = "nodes.map";
private const string WayMapRegionFileName = "ways.map";
public Graph.Graph? LoadRegionFromRegionId(long regionId)
{
logger?.LogDebug($"Loading Region {regionId}");
Graph.Graph ret = new();
StreamReader nodesReader, waysReader;
string nodesPath = Path.Join(ImportFolderPath, Converter.NodesRegionDirectory, regionId.ToString());
string waysPath = Path.Join(ImportFolderPath, Converter.WaysRegionDirectory, regionId.ToString());
try
{
nodesReader = new(nodesPath, Encoding.UTF8);
}
catch (Exception e)
{
logger?.LogError(e, $"Could not find Region in Nodes-Directory. ({nodesPath})");
return null;
}
try
{
waysReader = new(waysPath, Encoding.UTF8);
}
catch (Exception e)
{
logger?.LogError(e, $"Could not find Region in Ways-Directory. ({waysPath})");
return null;
}
logger?.LogDebug($"Loading Nodes {nodesPath}");
while (!nodesReader.EndOfStream)
{
string? line = nodesReader.ReadLine();
if(line is null)
continue;
Node n = Node.Deserialize(line);
ret.Nodes.Add(n.ID, new Graph.Node(n.Lat, n.Lon));
}
logger?.LogDebug($"Loading Ways {waysPath}");
while (!waysReader.EndOfStream)
{
string? line = waysReader.ReadLine();
if(line is null)
continue;
Way w = Way.Deserialize(line);
ret.Ways.Add(w.ID, new Graph.Way(w.Tags));
for (int i = 1; i < w.NodeIds.Count; i++)
{
ulong node1Id = w.NodeIds[i - 1];
ulong node2Id = w.NodeIds[i];
if (ret.Nodes.TryGetValue(node1Id, out Graph.Node? node1) && ret.Nodes.TryGetValue(node2Id, out Graph.Node? node2))
{
node1.Neighbors.TryAdd(node2Id, w.ID);
node2.Neighbors.TryAdd(node1Id, w.ID);
//TODO add oneway checks
}
}
}
return ret;
}
public Graph.Graph? LoadRegionFromNodeId(ulong nodeId)
{
logger?.LogDebug($"Loading Region for Node {nodeId}");
StreamReader nodesMapFileStream = new(Path.Join(ImportFolderPath, NodesMapRegionFileName), Encoding.ASCII);
while (!nodesMapFileStream.EndOfStream)
{
string? line = nodesMapFileStream.ReadLine();
if(line is null)
continue;
try
{
ulong id = ulong.Parse(line.Split('-')[0]);
if (id == nodeId)
return LoadRegionFromRegionId(long.Parse(line.Split('-')[1]));
}
catch (Exception e)
{
logger?.LogError(e, "Error parsing Node-line.");
}
}
logger?.LogWarning($"Could not find Node {nodeId}");
return null;
}
public Graph.Graph?[] LoadRegionsFromWayId(ulong wayId)
{
logger?.LogDebug($"Loading Region for Way {wayId}");
StreamReader waysMapFileStream = new(Path.Join(ImportFolderPath, WayMapRegionFileName), Encoding.ASCII);
while (!waysMapFileStream.EndOfStream)
{
string? line = waysMapFileStream.ReadLine();
if(line is null)
continue;
try
{
ulong id = ulong.Parse(line.Split('-')[0]);
if (id == wayId)
return line.Split('-')[1]
.Split(',')
.Select(rId => LoadRegionFromRegionId(long.Parse(rId)))
.ToArray();
}
catch (Exception e)
{
logger?.LogError(e, "Error parsing Way-line.");
}
}
logger?.LogWarning($"Could not find Way {wayId}");
return [];
}
public Graph.Graph? LoadRegionFromCoordinates(float lat, float lon)
{
logger?.LogDebug($"Loading Region for Coordinates {lat} {lon}");
return LoadRegionFromRegionId(Utils.RegionUtils.GetRegionIdFromCoordinates(lat, lon, regionSize));
}
}

18
OSM_Regions/Utils/RegionUtils.cs Normal file
View File

@ -0,0 +1,18 @@
using System.Globalization;
namespace OSM_Regions.Utils;
public static class RegionUtils
{
public static long GetRegionIdFromCoordinates(float lat, float lon, float regionSize)
{
string latStr = $"{Math.Floor(lat / regionSize):000000}".Replace(".","").Replace(",", "");
string lonStr = $"{Math.Floor(lon / regionSize):000000}".Replace(".","").Replace(",", "");
return long.Parse(string.Concat(latStr, lonStr));
}
public static long GetRegionIdFromCoordinates(string lat, string lon, float regionSize) =>
GetRegionIdFromCoordinates(float.Parse(lat, NumberStyles.Float, NumberFormatInfo.InvariantInfo),
float.Parse(lon, NumberStyles.Float, NumberFormatInfo.InvariantInfo),
regionSize);
}