glax/OSMServer
1
0
Fork 0
Code Pull Requests Releases Wiki Activity

Rewrote XmlImporter to Write Region-files while reading from Stream

Browse Source
This commit is contained in:
glax 2023-03-31 21:56:27 +02:00
parent 07ff9602b8
commit b621ff632b
5 changed files with 184 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Pathfinding/RegionManager.cs
View File

@ -6,7 +6,7 @@ namespace OSMImporter
public class RegionManager
{
private string workingDirectory { get; }
private readonly Dictionary<int, Region> _regions = new();
private readonly Dictionary<ulong, Region> _regions = new();
public RegionManager(string workingDirectory)
{
@ -44,23 +44,7 @@ namespace OSMImporter
/// <exception cref="FileNotFoundException">If the Regionfile can not be found.</exception>
private Region LoadRegion(Coordinates coordinates)
{
string fullPath = Path.Combine(workingDirectory, Coordinates.GetRegionHashCode(coordinates).ToString());
DateTime startTime = DateTime.Now;
if (!File.Exists(fullPath))
{
throw new FileNotFoundException(string.Format("[{0}] Region does not exist: {1}", startTime, fullPath));
}
FileStream fileStream = new FileStream(fullPath, FileMode.Open);
long fileStreamLength = fileStream.Length;
Console.WriteLine("[{0}] Loading [{1}]bytes from {2}", startTime.ToLocalTime(), fileStreamLength, fullPath);
byte[] regionBytes = new byte[fileStream.Length];
int loadedBytesLength = fileStream.Read(regionBytes, 0, regionBytes.Length);
fileStream.Close();
Console.WriteLine("\tLoaded [{0}]bytes ({1:P1}) in [{2}]ms", loadedBytesLength, fileStreamLength / loadedBytesLength,DateTime.Now.Subtract(startTime).TotalMilliseconds);
return ByteConverter.ToRegion(regionBytes);
throw new NotImplementedException();
}
public OsmNode? GetNode(Coordinates coordinates)

202
Server/RegionConverter.cs
View File

@ -1,6 +1,7 @@
using System.Globalization;
using System.Runtime.Serialization.Formatters.Binary;
using System.Xml;
using OSMDatastructure;
using OSMDatastructure.Graph;
namespace Server;
@ -15,58 +16,207 @@ public class RegionConverter
{
NumberDecimalSeparator = "."
};
private const string NodesFileName = "region.nodes";
private const string WaysFileName = "region.ways";
private const string tagsFileName = ".tags";
public static HashSet<Region> ImportXml(string filePath)
public static void ConvertXMLToRegions(string filePath, string outputPath)
{
if (!File.Exists(filePath))
throw new FileNotFoundException();
if (!Directory.Exists(outputPath))
Directory.CreateDirectory(outputPath);
Console.WriteLine("Getting highwayNodeIds...");
Console.WriteLine("Converting Nodes...");
FileStream xmlFileStream = new FileStream(filePath, FileMode.Open);
Dictionary<ulong, ulong> nodeIdRegionDict = GetNodesAndRegions(XmlReader.Create(xmlFileStream, ReaderSettings));
xmlFileStream.Position = 0;
Console.WriteLine("Importing Nodes...");
Dictionary<ulong, OsmNode> nodes =
GetHighwayNodesFromIds(XmlReader.Create(xmlFileStream, ReaderSettings), requiredNodeIds);
requiredNodeIds.Clear();
Dictionary<ulong, ulong> nodeIdRegionDict = GetNodesAndRegions(XmlReader.Create(xmlFileStream, ReaderSettings), outputPath);
xmlFileStream.Position = 0;
Console.WriteLine("Importing Ways...");
HashSet<OsmNode> retNodes = ConnectNodes(XmlReader.Create(xmlFileStream, ReaderSettings), nodes);
nodes.Clear();
return retNodes;
Console.WriteLine("Converting Ways...");
ImportWays(XmlReader.Create(xmlFileStream, ReaderSettings), nodeIdRegionDict, outputPath);
}
private static Dictionary<ulong, ulong> GetNodesAndRegions(XmlReader xmlReader)
private static Dictionary<ulong, ulong> GetNodesAndRegions(XmlReader xmlReader, string outputPath)
{
BinaryFormatter bFormatter = new BinaryFormatter();
Dictionary<ulong, ulong> nodeRegions = new();
Dictionary<ulong, FileStream> regionFileStreams = new();
Dictionary<ulong, OsmNode> tmpAllNodes = new();
bool isHighway = false;
HashSet<ulong> currentIds = new();
while (xmlReader.Read())
{
if (xmlReader.Name == "node")
{
ulong id = Convert.ToUInt64(xmlReader.GetAttribute("id")!);
float lat = Convert.ToSingle(xmlReader.GetAttribute("lat")!, decimalInfo);
float lon = Convert.ToSingle(xmlReader.GetAttribute("lon")!, decimalInfo);
tmpAllNodes.TryAdd(id, new OsmNode(id, lat, lon));
}
else if (xmlReader.Name == "way")
{
isHighway = false;
currentIds.Clear();
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway"))
{
isHighway = true;
}
else if (xmlReader.Name == "nd")
{
try
{
ulong id = Convert.ToUInt64(xmlReader.GetAttribute("ref")!);
currentIds.Add(id);
}
catch (FormatException) { };
}
}
wayReader.Close();
foreach (ulong nodeId in currentIds.Where(key => tmpAllNodes.ContainsKey(key)))
{
if (isHighway)
{
ulong regionHash = Coordinates.GetRegionHashCode(tmpAllNodes[nodeId].coordinates);
FileStream nodesRegionStream;
if(regionFileStreams.ContainsKey(regionHash))
nodesRegionStream = regionFileStreams[regionHash];
else
{
string regionPath = Path.Combine(outputPath, regionHash.ToString());
Directory.CreateDirectory(regionPath);
string nodesRegionPath = Path.Combine(regionPath, NodesFileName);
nodesRegionStream = new FileStream(nodesRegionPath, FileMode.Create);
regionFileStreams.Add(regionHash, nodesRegionStream);
}
nodeRegions.Add(nodeId, regionHash);
#pragma warning disable SYSLIB0011 //eheheh
bFormatter.Serialize(nodesRegionStream, tmpAllNodes[nodeId]);
#pragma warning restore SYSLIB0011
}
tmpAllNodes.Remove(nodeId);
}
}
}
xmlReader.Close();
return nodeRegions;
}
private static void ImportWays(XmlReader xmlReader, Dictionary<ulong, ulong> nodeRegions, string outputPath)
{
BinaryFormatter bFormatter = new BinaryFormatter();
bool currentWayIsHighway;
ulong currentWayId = 0;
List<ulong> currentNodeIds = new();
Dictionary<Tag.TagType, dynamic> currentTags = new();
Dictionary<ulong, FileStream> regionWaysFileStreams = new();
string wayFileStreamPath = "";
FileStream wayFileStream = new FileStream(Path.GetTempFileName(), FileMode.Create);
while (xmlReader.ReadToFollowing("way"))
{
isHighway = false;
currentIds.Clear();
ulong wayId = Convert.ToUInt64(xmlReader.GetAttribute("id")!);
currentWayIsHighway = false;
currentNodeIds.Clear();
currentTags.Clear();
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway"))
currentWayId = Convert.ToUInt64(wayReader.GetAttribute("id")!);
if (wayReader.Name == "tag")
{
isHighway = true;
Tag wayTag = Tag.ConvertToTag(wayReader.GetAttribute("k")!, wayReader.GetAttribute("v")!);
currentTags.TryAdd(wayTag.key, wayTag.value);
if(wayTag.key == Tag.TagType.highway)
currentWayIsHighway = true;
}
else if (xmlReader.Name == "nd")
else if (wayReader.Name == "nd")
{
try
{
currentIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
ulong nodeId = Convert.ToUInt64(wayReader.GetAttribute("ref"));
currentNodeIds.Add(nodeId);
}
catch (FormatException) { };
}
}
if (isHighway)
{
retSet.UnionWith(currentIds);
}
wayReader.Close();
if (currentWayIsHighway)
{
for (int i = 0; i < currentNodeIds.Count - 1; i++)
{
ulong node1Id = currentNodeIds[i];
ulong node2Id = currentNodeIds[i+1];
if (currentTags.ContainsKey(Tag.TagType.oneway) && (bool)currentTags[Tag.TagType.oneway] && nodeRegions.ContainsKey(node1Id) && nodeRegions.ContainsKey(node2Id))
{
if (currentTags.ContainsKey(Tag.TagType.forward) && !(bool)currentTags[Tag.TagType.forward])
{
OsmWay n21e = new OsmWay(currentWayId, node2Id, node1Id, nodeRegions[node2Id]);
if (!regionWaysFileStreams.ContainsKey(nodeRegions[node2Id]))
{
string waysRegionPath = Path.Combine(outputPath, nodeRegions[node2Id].ToString(), WaysFileName);
regionWaysFileStreams.Add(nodeRegions[node2Id], new FileStream(waysRegionPath, FileMode.OpenOrCreate));
}
#pragma warning disable SYSLIB0011
bFormatter.Serialize(regionWaysFileStreams[nodeRegions[node2Id]], n21e);
wayFileStreamPath = Path.Combine(outputPath, nodeRegions[node2Id].ToString(), $"{wayId}.{tagsFileName}");
wayFileStream = new FileStream(wayFileStreamPath, FileMode.OpenOrCreate);
bFormatter.Serialize(wayFileStream, currentTags);
wayFileStream.Dispose();
}
else
{
OsmWay n12e = new OsmWay(currentWayId, node1Id, node2Id, nodeRegions[node2Id]);
if (!regionWaysFileStreams.ContainsKey(nodeRegions[node1Id]))
{
string waysRegionPath = Path.Combine(outputPath, nodeRegions[node1Id].ToString(), WaysFileName);
regionWaysFileStreams.Add(nodeRegions[node1Id], new FileStream(waysRegionPath, FileMode.OpenOrCreate));
}
bFormatter.Serialize(regionWaysFileStreams[nodeRegions[node1Id]], n12e);
wayFileStreamPath = Path.Combine(outputPath, nodeRegions[node1Id].ToString(), $"{wayId}.{tagsFileName}");
wayFileStream = new FileStream(wayFileStreamPath, FileMode.OpenOrCreate);
bFormatter.Serialize(wayFileStream, currentTags);
wayFileStream.Dispose();
}
}
else if(nodeRegions.ContainsKey(node1Id) && nodeRegions.ContainsKey(node2Id))
{
OsmWay n12e = new OsmWay(currentWayId, node1Id, node2Id, nodeRegions[node2Id]);
if (!regionWaysFileStreams.ContainsKey(nodeRegions[node1Id]))
{
string waysRegionPath = Path.Combine(outputPath, nodeRegions[node1Id].ToString(), WaysFileName);
regionWaysFileStreams.Add(nodeRegions[node1Id], new FileStream(waysRegionPath, FileMode.OpenOrCreate));
}
bFormatter.Serialize(regionWaysFileStreams[nodeRegions[node1Id]], n12e);
wayFileStreamPath = Path.Combine(outputPath, nodeRegions[node1Id].ToString(), $"{wayId}.{tagsFileName}");
wayFileStream = new FileStream(wayFileStreamPath, FileMode.OpenOrCreate);
bFormatter.Serialize(wayFileStream, currentTags);
wayFileStream.Dispose();
OsmWay n21e = new OsmWay(currentWayId, node2Id, node1Id, nodeRegions[node2Id]);
if (!regionWaysFileStreams.ContainsKey(nodeRegions[node2Id]))
{
string waysRegionPath = Path.Combine(outputPath, nodeRegions[node2Id].ToString(), WaysFileName);
regionWaysFileStreams.Add(nodeRegions[node2Id], new FileStream(waysRegionPath, FileMode.OpenOrCreate));
}
bFormatter.Serialize(regionWaysFileStreams[nodeRegions[node1Id]], n21e);
wayFileStreamPath = Path.Combine(outputPath, nodeRegions[node2Id].ToString(), $"{wayId}.{tagsFileName}");
wayFileStream = new FileStream(wayFileStreamPath, FileMode.OpenOrCreate);
bFormatter.Serialize(wayFileStream, currentTags);
wayFileStream.Dispose();
#pragma warning restore SYSLIB0011
}
}
}
}
xmlReader.Close();
}

27
Server/Server.cs
View File

@ -1,8 +1,3 @@
using OSMDatastructure;
using OSMDatastructure.Graph;
using Pathfinding;
namespace Server;
public class Server
@ -10,9 +5,12 @@ public class Server
public static void Main(string[] args)
{
HashSet<OsmNode> nodes = XmlImporter.ImportXml("D:/stuttgart-regbez-latest.osm/stuttgart-regbez-latest.osm");
HashSet<Region> regions = XmlImporter.SplitIntoRegions(nodes);
WriteRegionsToFile(regions, "D:/stuttgart-regbez");
ConsoleWriter newConsole = new ConsoleWriter();
Console.SetOut(newConsole);
Console.SetError(newConsole);
RegionConverter.ConvertXMLToRegions("D:/stuttgart-regbez-latest.osm", "D:/stuttgart-regbez-latest");
//RegionConverter.ConvertXMLToRegions("D:/map.osm", "D:/map");
/*
Coordinates start = new Coordinates(48.243351f, 11.640417f);
@ -31,17 +29,4 @@ public class Server
}
Console.WriteLine();*/
}
private static void WriteRegionsToFile(HashSet<Region> regions, string outputFolderPath)
{
Console.WriteLine(string.Format("[{0}] Writing files...", DateTime.Now.ToLocalTime()));
Directory.CreateDirectory(outputFolderPath);
foreach (Region region in regions)
{
FileStream regionFileStream =
new FileStream(Path.Combine(outputFolderPath, region.regionHash.ToString()), FileMode.Create);
regionFileStream.Write(ByteConverter.GetBytes(region));
regionFileStream.Close();
}
}
}

1
Server/Server.csproj
View File

@ -8,7 +8,6 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OsmXmlToRegionConverter\OsmXmlToRegionConverter.csproj" />
<ProjectReference Include="..\Pathfinding\Pathfinding.csproj" />
</ItemGroup>

171
Server/XmlImporter.cs
View File

@ -1,171 +0,0 @@
using System.Globalization;
using System.Xml;
using OSMDatastructure;
using OSMDatastructure.Graph;
namespace Server;
public static class XmlImporter
{
private static readonly XmlReaderSettings readerSettings = new()
{
IgnoreWhitespace = true,
IgnoreComments = true
};
public static HashSet<OsmNode> ImportXml(string filePath)
{
if (!File.Exists(filePath))
throw new FileNotFoundException();
Console.WriteLine(string.Format("[{0}] Getting highwayNodeIds...", DateTime.Now.ToLocalTime()));
FileStream xmlFileStream = new FileStream(filePath, FileMode.Open);
HashSet<ulong> requiredNodeIds = GetHighwayNodeIds(XmlReader.Create(xmlFileStream, readerSettings));
xmlFileStream.Position = 0;
Console.WriteLine(string.Format("[{0}] Importing Nodes...", DateTime.Now.ToLocalTime()));
Dictionary<ulong, OsmNode> nodes =
GetHighwayNodesFromIds(XmlReader.Create(xmlFileStream, readerSettings), requiredNodeIds);
requiredNodeIds.Clear();
xmlFileStream.Position = 0;
Console.WriteLine(string.Format("[{0}] Importing Ways...", DateTime.Now.ToLocalTime()));
HashSet<OsmNode> retNodes = ConnectNodes(XmlReader.Create(xmlFileStream, readerSettings), nodes);
nodes.Clear();
return retNodes;
}
public static HashSet<Region> SplitIntoRegions(HashSet<OsmNode> nodes)
{
Console.WriteLine(string.Format("[{0}] Splitting into Regions...", DateTime.Now.ToLocalTime()));
Dictionary<int, Region> retRegions = new();
foreach (OsmNode node in nodes)
{
int regionHash = Coordinates.GetRegionHashCode(node.coordinates);
if(retRegions.ContainsKey(regionHash))
{
retRegions[regionHash].nodes.Add(node);
}
else
{
Region newRegion = new Region(regionHash);
newRegion.nodes.Add(node);
retRegions.Add(regionHash, newRegion);
}
}
return retRegions.Values.ToHashSet();
}
private static readonly NumberFormatInfo decimalInfo = new()
{
NumberDecimalSeparator = "."
};
public static HashSet<ulong> GetHighwayNodeIds(XmlReader xmlReader)
{
HashSet<ulong> retSet = new();
bool isHighway;
HashSet<ulong> currentIds = new();
while (xmlReader.ReadToFollowing("way"))
{
isHighway = false;
currentIds.Clear();
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway"))
{
isHighway = true;
}
else if (xmlReader.Name == "nd")
{
try
{
currentIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
}
catch (FormatException) { };
}
}
if (isHighway)
{
retSet.UnionWith(currentIds);
}
wayReader.Close();
}
xmlReader.Close();
return retSet;
}
private static Dictionary<ulong, OsmNode> GetHighwayNodesFromIds(XmlReader xmlReader, HashSet<ulong> ids)
{
Dictionary<ulong, OsmNode> retDict = new();
while (xmlReader.ReadToFollowing("node"))
{
ulong id = Convert.ToUInt64(xmlReader.GetAttribute("id"));
if (ids.Contains(id))
{
float lat = Convert.ToSingle(xmlReader.GetAttribute("lat")!, decimalInfo);
float lon = Convert.ToSingle(xmlReader.GetAttribute("lon")!, decimalInfo);
retDict.Add(id, new OsmNode(lat, lon));
}
}
xmlReader.Close();
return retDict;
}
private static HashSet<OsmNode> ConnectNodes(XmlReader xmlReader, Dictionary<ulong, OsmNode> dict)
{
while (xmlReader.ReadToFollowing("way"))
{
Dictionary<OsmEdge.tagType, object> tags = new();
List<ulong> wayNodeIds = new();
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
if (xmlReader.Name == "tag")
{
KeyValuePair<OsmEdge.tagType, object> tag =
OsmEdge.ConvertToTag(wayReader.GetAttribute("k")!, wayReader.GetAttribute("v")!);
tags.TryAdd(tag.Key, tag.Value);
}
else if (xmlReader.Name == "nd")
{
wayNodeIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
}
}
if (tags.ContainsKey(OsmEdge.tagType.highway))
{
ulong[] ids = wayNodeIds.Where(dict.ContainsKey).ToArray();
for (int i = 0; i < ids.Length - 1; i++)
{
OsmNode n1 = dict[ids[i]];
OsmNode n2 = dict[ids[i + 1]];
if (tags.ContainsKey(OsmEdge.tagType.oneway) && (bool)tags[OsmEdge.tagType.oneway])
{
if (tags.ContainsKey(OsmEdge.tagType.forward) && !(bool)tags[OsmEdge.tagType.forward])
{
n2.edges.Add(new OsmEdge(n1.coordinates, tags));
}
else
{
n1.edges.Add(new OsmEdge(n2.coordinates, tags));
}
}
else
{
n1.edges.Add(new OsmEdge(n2.coordinates, tags));
n2.edges.Add(new OsmEdge(n1.coordinates, tags));
}
}
}
wayReader.Close();
}
xmlReader.Close();
return dict.Values.ToHashSet();
}
}