Compare commits

..

6 Commits

Author SHA1 Message Date
b621ff632b Rewrote XmlImporter to Write Region-files while reading from Stream 2023-03-31 21:56:27 +02:00
07ff9602b8 Temporary out of order 2023-03-31 21:55:54 +02:00
ac77708834 Renamed OsmEdge to OsmWay and added startnodeid (for serialization) 2023-03-31 21:55:43 +02:00
255d924dc4 Made GetRegionHashCode(Coordinates) wrapper
public static ulong GetRegionHashCode(Coordinates coordinates) now calls GetRegionHashCode(coordinates.latitude, coordinates.longitude)
2023-03-31 21:55:12 +02:00
0d4597c12c Made Region Serializable 2023-03-31 21:54:32 +02:00
6a2ddf3277 Added way to add Tags from KeyValuePair 2023-03-31 21:54:01 +02:00
11 changed files with 206 additions and 252 deletions

View File

@ -32,9 +32,7 @@ public class Coordinates
public static ulong GetRegionHashCode(Coordinates coordinates)
{
float latRegion = coordinates.latitude - coordinates.latitude % Region.RegionSize;
float lonRegion = coordinates.longitude - coordinates.longitude % Region.RegionSize;
return GetHashCode(latRegion, lonRegion);
return GetRegionHashCode(coordinates.latitude, coordinates.longitude);
}
private const float decimalCoordsSave = 10000; //Latitude maxChars = 7

View File

@ -4,7 +4,7 @@ namespace OSMDatastructure.Graph;
public class OsmNode
{
public ulong nodeId { get; }
public HashSet<OsmEdge> edges { get; }
public HashSet<OsmWay> edges { get; }
public Coordinates coordinates { get; }
[NonSerialized]public OsmNode? previousPathNode = null;
@ -26,9 +26,9 @@ public class OsmNode
this.coordinates = coordinates;
}
public OsmEdge? GetEdgeToNode(OsmNode n)
public OsmWay? GetEdgeToNode(OsmNode n)
{
foreach (OsmEdge e in this.edges)
foreach (OsmWay e in this.edges)
if (e.neighborId == n.nodeId)
return e;
return null;

View File

@ -1,17 +1,19 @@
namespace OSMDatastructure.Graph;
[Serializable]
public class OsmEdge
public class OsmWay
{
public ulong wayId { get; }
public ulong startId { get; }
public ulong neighborId { get; }
public ulong neighborRegion { get; }
public OsmEdge(ulong wayID, ulong neighborID, ulong neighborRegion)
public OsmWay(ulong wayId, ulong startId, ulong neighborId, ulong neighborRegion)
{
this.wayId = wayID;
this.neighborId = neighborID;
this.wayId = wayId;
this.startId = startId;
this.neighborId = neighborId;
this.neighborRegion = neighborRegion;
}

View File

@ -5,7 +5,7 @@ namespace OSMDatastructure;
[Serializable]
public class Region
{
[NonSerialized]public const float RegionSize = 0.01f;
[NonSerialized]public const float RegionSize = 0.1f;
public readonly HashSet<OsmNode> nodes = new();
public ulong regionHash { get; }
public TagManager tagManager { get; }

View File

@ -33,6 +33,13 @@ public class TagManager
}
}
public void AddTag(ulong wayId, KeyValuePair<Tag.TagType, dynamic> keyValuePair)
{
if(!wayTags.ContainsKey(wayId))
wayTags.Add(wayId, new HashSet<Tag>());
wayTags[wayId].Add(new Tag(keyValuePair.Key, keyValuePair.Value));
}
public HashSet<Tag>? GetTagsForWayId(ulong wayId)
{
return wayTags.TryGetValue(wayId, out HashSet<Tag>? value) ? value : null;

View File

@ -6,8 +6,8 @@ namespace Pathfinding;
public class Pathfinder
{
public static List<OsmNode> CustomAStar(string workingDir, Coordinates start, Coordinates goal, OsmEdge.speedType vehicle)
/*
public static List<OsmNode> CustomAStar(string workingDir, Coordinates start, Coordinates goal, OsmWay.speedType vehicle)
{
RegionManager regionManager = new RegionManager(workingDir);
Region startRegion, goalRegion;
@ -48,7 +48,7 @@ public class Pathfinder
}
}
foreach (OsmEdge edge in closestNodeToGoal.edges)
foreach (OsmWay edge in closestNodeToGoal.edges)
{
OsmNode? neighbor = regionManager.GetNode(edge.neighborCoordinates);
if (neighbor != null)
@ -100,5 +100,5 @@ public class Pathfinder
}
return closest;
}
}*/
}

View File

@ -6,7 +6,7 @@ namespace OSMImporter
public class RegionManager
{
private string workingDirectory { get; }
private readonly Dictionary<int, Region> _regions = new();
private readonly Dictionary<ulong, Region> _regions = new();
public RegionManager(string workingDirectory)
{
@ -44,23 +44,7 @@ namespace OSMImporter
/// <exception cref="FileNotFoundException">If the Regionfile can not be found.</exception>
private Region LoadRegion(Coordinates coordinates)
{
string fullPath = Path.Combine(workingDirectory, Coordinates.GetRegionHashCode(coordinates).ToString());
DateTime startTime = DateTime.Now;
if (!File.Exists(fullPath))
{
throw new FileNotFoundException(string.Format("[{0}] Region does not exist: {1}", startTime, fullPath));
}
FileStream fileStream = new FileStream(fullPath, FileMode.Open);
long fileStreamLength = fileStream.Length;
Console.WriteLine("[{0}] Loading [{1}]bytes from {2}", startTime.ToLocalTime(), fileStreamLength, fullPath);
byte[] regionBytes = new byte[fileStream.Length];
int loadedBytesLength = fileStream.Read(regionBytes, 0, regionBytes.Length);
fileStream.Close();
Console.WriteLine("\tLoaded [{0}]bytes ({1:P1}) in [{2}]ms", loadedBytesLength, fileStreamLength / loadedBytesLength,DateTime.Now.Subtract(startTime).TotalMilliseconds);
return ByteConverter.ToRegion(regionBytes);
throw new NotImplementedException();
}
public OsmNode? GetNode(Coordinates coordinates)

View File

@ -1,6 +1,7 @@
using System.Globalization;
using System.Runtime.Serialization.Formatters.Binary;
using System.Xml;
using OSMDatastructure;
using OSMDatastructure.Graph;
namespace Server;
@ -16,33 +17,44 @@ public class RegionConverter
NumberDecimalSeparator = "."
};
public static HashSet<Region> ImportXml(string filePath)
private const string NodesFileName = "region.nodes";
private const string WaysFileName = "region.ways";
private const string tagsFileName = ".tags";
public static void ConvertXMLToRegions(string filePath, string outputPath)
{
if (!File.Exists(filePath))
throw new FileNotFoundException();
if (!Directory.Exists(outputPath))
Directory.CreateDirectory(outputPath);
Console.WriteLine("Getting highwayNodeIds...");
Console.WriteLine("Converting Nodes...");
FileStream xmlFileStream = new FileStream(filePath, FileMode.Open);
Dictionary<ulong, ulong> nodeIdRegionDict = GetNodesAndRegions(XmlReader.Create(xmlFileStream, ReaderSettings));
Dictionary<ulong, ulong> nodeIdRegionDict = GetNodesAndRegions(XmlReader.Create(xmlFileStream, ReaderSettings), outputPath);
xmlFileStream.Position = 0;
Console.WriteLine("Importing Nodes...");
Dictionary<ulong, OsmNode> nodes =
GetHighwayNodesFromIds(XmlReader.Create(xmlFileStream, ReaderSettings), requiredNodeIds);
requiredNodeIds.Clear();
xmlFileStream.Position = 0;
Console.WriteLine("Importing Ways...");
HashSet<OsmNode> retNodes = ConnectNodes(XmlReader.Create(xmlFileStream, ReaderSettings), nodes);
nodes.Clear();
return retNodes;
Console.WriteLine("Converting Ways...");
ImportWays(XmlReader.Create(xmlFileStream, ReaderSettings), nodeIdRegionDict, outputPath);
}
private static Dictionary<ulong, ulong> GetNodesAndRegions(XmlReader xmlReader)
private static Dictionary<ulong, ulong> GetNodesAndRegions(XmlReader xmlReader, string outputPath)
{
BinaryFormatter bFormatter = new BinaryFormatter();
Dictionary<ulong, ulong> nodeRegions = new();
Dictionary<ulong, FileStream> regionFileStreams = new();
Dictionary<ulong, OsmNode> tmpAllNodes = new();
bool isHighway = false;
while (xmlReader.ReadToFollowing("way"))
HashSet<ulong> currentIds = new();
while (xmlReader.Read())
{
if (xmlReader.Name == "node")
{
ulong id = Convert.ToUInt64(xmlReader.GetAttribute("id")!);
float lat = Convert.ToSingle(xmlReader.GetAttribute("lat")!, decimalInfo);
float lon = Convert.ToSingle(xmlReader.GetAttribute("lon")!, decimalInfo);
tmpAllNodes.TryAdd(id, new OsmNode(id, lat, lon));
}
else if (xmlReader.Name == "way")
{
isHighway = false;
currentIds.Clear();
@ -57,16 +69,154 @@ public class RegionConverter
{
try
{
currentIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
ulong id = Convert.ToUInt64(xmlReader.GetAttribute("ref")!);
currentIds.Add(id);
}
catch (FormatException) { };
}
}
wayReader.Close();
foreach (ulong nodeId in currentIds.Where(key => tmpAllNodes.ContainsKey(key)))
{
if (isHighway)
{
retSet.UnionWith(currentIds);
ulong regionHash = Coordinates.GetRegionHashCode(tmpAllNodes[nodeId].coordinates);
FileStream nodesRegionStream;
if(regionFileStreams.ContainsKey(regionHash))
nodesRegionStream = regionFileStreams[regionHash];
else
{
string regionPath = Path.Combine(outputPath, regionHash.ToString());
Directory.CreateDirectory(regionPath);
string nodesRegionPath = Path.Combine(regionPath, NodesFileName);
nodesRegionStream = new FileStream(nodesRegionPath, FileMode.Create);
regionFileStreams.Add(regionHash, nodesRegionStream);
}
nodeRegions.Add(nodeId, regionHash);
#pragma warning disable SYSLIB0011 //eheheh
bFormatter.Serialize(nodesRegionStream, tmpAllNodes[nodeId]);
#pragma warning restore SYSLIB0011
}
tmpAllNodes.Remove(nodeId);
}
}
}
xmlReader.Close();
return nodeRegions;
}
private static void ImportWays(XmlReader xmlReader, Dictionary<ulong, ulong> nodeRegions, string outputPath)
{
BinaryFormatter bFormatter = new BinaryFormatter();
bool currentWayIsHighway;
ulong currentWayId = 0;
List<ulong> currentNodeIds = new();
Dictionary<Tag.TagType, dynamic> currentTags = new();
Dictionary<ulong, FileStream> regionWaysFileStreams = new();
string wayFileStreamPath = "";
FileStream wayFileStream = new FileStream(Path.GetTempFileName(), FileMode.Create);
while (xmlReader.ReadToFollowing("way"))
{
ulong wayId = Convert.ToUInt64(xmlReader.GetAttribute("id")!);
currentWayIsHighway = false;
currentNodeIds.Clear();
currentTags.Clear();
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
currentWayId = Convert.ToUInt64(wayReader.GetAttribute("id")!);
if (wayReader.Name == "tag")
{
Tag wayTag = Tag.ConvertToTag(wayReader.GetAttribute("k")!, wayReader.GetAttribute("v")!);
currentTags.TryAdd(wayTag.key, wayTag.value);
if(wayTag.key == Tag.TagType.highway)
currentWayIsHighway = true;
}
else if (wayReader.Name == "nd")
{
try
{
ulong nodeId = Convert.ToUInt64(wayReader.GetAttribute("ref"));
currentNodeIds.Add(nodeId);
}
catch (FormatException) { };
}
}
wayReader.Close();
if (currentWayIsHighway)
{
for (int i = 0; i < currentNodeIds.Count - 1; i++)
{
ulong node1Id = currentNodeIds[i];
ulong node2Id = currentNodeIds[i+1];
if (currentTags.ContainsKey(Tag.TagType.oneway) && (bool)currentTags[Tag.TagType.oneway] && nodeRegions.ContainsKey(node1Id) && nodeRegions.ContainsKey(node2Id))
{
if (currentTags.ContainsKey(Tag.TagType.forward) && !(bool)currentTags[Tag.TagType.forward])
{
OsmWay n21e = new OsmWay(currentWayId, node2Id, node1Id, nodeRegions[node2Id]);
if (!regionWaysFileStreams.ContainsKey(nodeRegions[node2Id]))
{
string waysRegionPath = Path.Combine(outputPath, nodeRegions[node2Id].ToString(), WaysFileName);
regionWaysFileStreams.Add(nodeRegions[node2Id], new FileStream(waysRegionPath, FileMode.OpenOrCreate));
}
#pragma warning disable SYSLIB0011
bFormatter.Serialize(regionWaysFileStreams[nodeRegions[node2Id]], n21e);
wayFileStreamPath = Path.Combine(outputPath, nodeRegions[node2Id].ToString(), $"{wayId}.{tagsFileName}");
wayFileStream = new FileStream(wayFileStreamPath, FileMode.OpenOrCreate);
bFormatter.Serialize(wayFileStream, currentTags);
wayFileStream.Dispose();
}
else
{
OsmWay n12e = new OsmWay(currentWayId, node1Id, node2Id, nodeRegions[node2Id]);
if (!regionWaysFileStreams.ContainsKey(nodeRegions[node1Id]))
{
string waysRegionPath = Path.Combine(outputPath, nodeRegions[node1Id].ToString(), WaysFileName);
regionWaysFileStreams.Add(nodeRegions[node1Id], new FileStream(waysRegionPath, FileMode.OpenOrCreate));
}
bFormatter.Serialize(regionWaysFileStreams[nodeRegions[node1Id]], n12e);
wayFileStreamPath = Path.Combine(outputPath, nodeRegions[node1Id].ToString(), $"{wayId}.{tagsFileName}");
wayFileStream = new FileStream(wayFileStreamPath, FileMode.OpenOrCreate);
bFormatter.Serialize(wayFileStream, currentTags);
wayFileStream.Dispose();
}
}
else if(nodeRegions.ContainsKey(node1Id) && nodeRegions.ContainsKey(node2Id))
{
OsmWay n12e = new OsmWay(currentWayId, node1Id, node2Id, nodeRegions[node2Id]);
if (!regionWaysFileStreams.ContainsKey(nodeRegions[node1Id]))
{
string waysRegionPath = Path.Combine(outputPath, nodeRegions[node1Id].ToString(), WaysFileName);
regionWaysFileStreams.Add(nodeRegions[node1Id], new FileStream(waysRegionPath, FileMode.OpenOrCreate));
}
bFormatter.Serialize(regionWaysFileStreams[nodeRegions[node1Id]], n12e);
wayFileStreamPath = Path.Combine(outputPath, nodeRegions[node1Id].ToString(), $"{wayId}.{tagsFileName}");
wayFileStream = new FileStream(wayFileStreamPath, FileMode.OpenOrCreate);
bFormatter.Serialize(wayFileStream, currentTags);
wayFileStream.Dispose();
OsmWay n21e = new OsmWay(currentWayId, node2Id, node1Id, nodeRegions[node2Id]);
if (!regionWaysFileStreams.ContainsKey(nodeRegions[node2Id]))
{
string waysRegionPath = Path.Combine(outputPath, nodeRegions[node2Id].ToString(), WaysFileName);
regionWaysFileStreams.Add(nodeRegions[node2Id], new FileStream(waysRegionPath, FileMode.OpenOrCreate));
}
bFormatter.Serialize(regionWaysFileStreams[nodeRegions[node1Id]], n21e);
wayFileStreamPath = Path.Combine(outputPath, nodeRegions[node2Id].ToString(), $"{wayId}.{tagsFileName}");
wayFileStream = new FileStream(wayFileStreamPath, FileMode.OpenOrCreate);
bFormatter.Serialize(wayFileStream, currentTags);
wayFileStream.Dispose();
#pragma warning restore SYSLIB0011
}
}
}
}
xmlReader.Close();
}

View File

@ -1,8 +1,3 @@
using OSMDatastructure;
using OSMDatastructure.Graph;
using Pathfinding;
namespace Server;
public class Server
@ -10,9 +5,12 @@ public class Server
public static void Main(string[] args)
{
HashSet<OsmNode> nodes = XmlImporter.ImportXml("D:/stuttgart-regbez-latest.osm/stuttgart-regbez-latest.osm");
HashSet<Region> regions = XmlImporter.SplitIntoRegions(nodes);
WriteRegionsToFile(regions, "D:/stuttgart-regbez");
ConsoleWriter newConsole = new ConsoleWriter();
Console.SetOut(newConsole);
Console.SetError(newConsole);
RegionConverter.ConvertXMLToRegions("D:/stuttgart-regbez-latest.osm", "D:/stuttgart-regbez-latest");
//RegionConverter.ConvertXMLToRegions("D:/map.osm", "D:/map");
/*
Coordinates start = new Coordinates(48.243351f, 11.640417f);
@ -31,17 +29,4 @@ public class Server
}
Console.WriteLine();*/
}
private static void WriteRegionsToFile(HashSet<Region> regions, string outputFolderPath)
{
Console.WriteLine(string.Format("[{0}] Writing files...", DateTime.Now.ToLocalTime()));
Directory.CreateDirectory(outputFolderPath);
foreach (Region region in regions)
{
FileStream regionFileStream =
new FileStream(Path.Combine(outputFolderPath, region.regionHash.ToString()), FileMode.Create);
regionFileStream.Write(ByteConverter.GetBytes(region));
regionFileStream.Close();
}
}
}

View File

@ -8,7 +8,6 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OsmXmlToRegionConverter\OsmXmlToRegionConverter.csproj" />
<ProjectReference Include="..\Pathfinding\Pathfinding.csproj" />
</ItemGroup>

View File

@ -1,171 +0,0 @@
using System.Globalization;
using System.Xml;
using OSMDatastructure;
using OSMDatastructure.Graph;
namespace Server;
public static class XmlImporter
{
private static readonly XmlReaderSettings readerSettings = new()
{
IgnoreWhitespace = true,
IgnoreComments = true
};
public static HashSet<OsmNode> ImportXml(string filePath)
{
if (!File.Exists(filePath))
throw new FileNotFoundException();
Console.WriteLine(string.Format("[{0}] Getting highwayNodeIds...", DateTime.Now.ToLocalTime()));
FileStream xmlFileStream = new FileStream(filePath, FileMode.Open);
HashSet<ulong> requiredNodeIds = GetHighwayNodeIds(XmlReader.Create(xmlFileStream, readerSettings));
xmlFileStream.Position = 0;
Console.WriteLine(string.Format("[{0}] Importing Nodes...", DateTime.Now.ToLocalTime()));
Dictionary<ulong, OsmNode> nodes =
GetHighwayNodesFromIds(XmlReader.Create(xmlFileStream, readerSettings), requiredNodeIds);
requiredNodeIds.Clear();
xmlFileStream.Position = 0;
Console.WriteLine(string.Format("[{0}] Importing Ways...", DateTime.Now.ToLocalTime()));
HashSet<OsmNode> retNodes = ConnectNodes(XmlReader.Create(xmlFileStream, readerSettings), nodes);
nodes.Clear();
return retNodes;
}
public static HashSet<Region> SplitIntoRegions(HashSet<OsmNode> nodes)
{
Console.WriteLine(string.Format("[{0}] Splitting into Regions...", DateTime.Now.ToLocalTime()));
Dictionary<int, Region> retRegions = new();
foreach (OsmNode node in nodes)
{
int regionHash = Coordinates.GetRegionHashCode(node.coordinates);
if(retRegions.ContainsKey(regionHash))
{
retRegions[regionHash].nodes.Add(node);
}
else
{
Region newRegion = new Region(regionHash);
newRegion.nodes.Add(node);
retRegions.Add(regionHash, newRegion);
}
}
return retRegions.Values.ToHashSet();
}
private static readonly NumberFormatInfo decimalInfo = new()
{
NumberDecimalSeparator = "."
};
public static HashSet<ulong> GetHighwayNodeIds(XmlReader xmlReader)
{
HashSet<ulong> retSet = new();
bool isHighway;
HashSet<ulong> currentIds = new();
while (xmlReader.ReadToFollowing("way"))
{
isHighway = false;
currentIds.Clear();
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway"))
{
isHighway = true;
}
else if (xmlReader.Name == "nd")
{
try
{
currentIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
}
catch (FormatException) { };
}
}
if (isHighway)
{
retSet.UnionWith(currentIds);
}
wayReader.Close();
}
xmlReader.Close();
return retSet;
}
private static Dictionary<ulong, OsmNode> GetHighwayNodesFromIds(XmlReader xmlReader, HashSet<ulong> ids)
{
Dictionary<ulong, OsmNode> retDict = new();
while (xmlReader.ReadToFollowing("node"))
{
ulong id = Convert.ToUInt64(xmlReader.GetAttribute("id"));
if (ids.Contains(id))
{
float lat = Convert.ToSingle(xmlReader.GetAttribute("lat")!, decimalInfo);
float lon = Convert.ToSingle(xmlReader.GetAttribute("lon")!, decimalInfo);
retDict.Add(id, new OsmNode(lat, lon));
}
}
xmlReader.Close();
return retDict;
}
private static HashSet<OsmNode> ConnectNodes(XmlReader xmlReader, Dictionary<ulong, OsmNode> dict)
{
while (xmlReader.ReadToFollowing("way"))
{
Dictionary<OsmEdge.tagType, object> tags = new();
List<ulong> wayNodeIds = new();
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
if (xmlReader.Name == "tag")
{
KeyValuePair<OsmEdge.tagType, object> tag =
OsmEdge.ConvertToTag(wayReader.GetAttribute("k")!, wayReader.GetAttribute("v")!);
tags.TryAdd(tag.Key, tag.Value);
}
else if (xmlReader.Name == "nd")
{
wayNodeIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
}
}
if (tags.ContainsKey(OsmEdge.tagType.highway))
{
ulong[] ids = wayNodeIds.Where(dict.ContainsKey).ToArray();
for (int i = 0; i < ids.Length - 1; i++)
{
OsmNode n1 = dict[ids[i]];
OsmNode n2 = dict[ids[i + 1]];
if (tags.ContainsKey(OsmEdge.tagType.oneway) && (bool)tags[OsmEdge.tagType.oneway])
{
if (tags.ContainsKey(OsmEdge.tagType.forward) && !(bool)tags[OsmEdge.tagType.forward])
{
n2.edges.Add(new OsmEdge(n1.coordinates, tags));
}
else
{
n1.edges.Add(new OsmEdge(n2.coordinates, tags));
}
}
else
{
n1.edges.Add(new OsmEdge(n2.coordinates, tags));
n2.edges.Add(new OsmEdge(n1.coordinates, tags));
}
}
}
wayReader.Close();
}
xmlReader.Close();
return dict.Values.ToHashSet();
}
}