Split into Datastructure, Added ByteConverter, Full Split & Load

This commit is contained in:
C9Glax 2023-02-03 21:13:51 +01:00
parent f31b1b577b
commit 311afcc02f
14 changed files with 332 additions and 187 deletions

View File

@ -1,8 +1,10 @@
namespace OSMImporter
namespace OSMDatastructure
{
public class Connection
{
public ulong endNodeId { get; }
public Coordinates endNodeCoordinates { get; }
private readonly Dictionary<tagType, object> _tags = new Dictionary<tagType, object>();
@ -11,14 +13,16 @@ namespace OSMImporter
highway, oneway, footway, sidewalk, cycleway, busway, forward, maxspeed, unknown
}
public Connection(ulong endNodeId)
public Connection(ulong endNodeId, Coordinates endNodeCoordinates)
{
this.endNodeId = endNodeId;
this.endNodeCoordinates = endNodeCoordinates;
}
public Connection(ulong endNodeId, Dictionary<string, string> tags)
public Connection(ulong endNodeId, Coordinates endNodeCoordinates, Dictionary<string, string> tags)
{
this.endNodeId = endNodeId;
this.endNodeCoordinates = endNodeCoordinates;
ImportTags(tags);
}
@ -117,29 +121,10 @@ namespace OSMImporter
return null;
}
}
/*
* Value 1: endId (4bytes)
* Value 2: tagsCount (1byte)
* Value x: tag + value (2bytes)
*/
public byte[] ToByte()
{
byte[] ret = new byte[sizeof(ulong) + 1 + this._tags.Count * 2];
int offset = 0;
Buffer.BlockCopy(new []{ this.endNodeId }, 0, ret, offset, sizeof(ulong) );
offset += sizeof(ulong);
byte tagsAmount = Convert.ToByte(this._tags.Count);
Buffer.SetByte(ret, offset++, tagsAmount);
foreach (KeyValuePair<tagType, object> tag in this._tags)
{
Buffer.BlockCopy(new byte[]{(byte)tag.Key, (byte)tag.Value}, 0, ret,offset, 2);
offset += 2;
}
return ret;
public Dictionary<tagType, object> GetTags()
{
return this._tags;
}
}
}

View File

@ -1,6 +1,6 @@
using System.Globalization;
namespace OSMImporter
namespace OSMDatastructure
{
public class Coordinates
{
@ -24,10 +24,10 @@ namespace OSMImporter
return latHash * offset + lonHash;
}
public ulong GetRegionHash()
public ulong GetRegionHash(float regionSize)
{
float latRegion = this.lat - this.lat % Importer.regionSize;
float lonRegion = this.lon - this.lon % Importer.regionSize;
float latRegion = this.lat - this.lat % regionSize;
float lonRegion = this.lon - this.lon % regionSize;
ulong latHash = Convert.ToUInt64((latRegion + 90) * decimalCoordsSave);
ulong lonHash = Convert.ToUInt64((lonRegion + 180) * decimalCoordsSave);
return latHash * offset + lonHash;

22
OSMDatastructure/Node.cs Normal file
View File

@ -0,0 +1,22 @@
namespace OSMDatastructure
{
public class Node : Coordinates
{
private readonly HashSet<Connection> _connections = new HashSet<Connection>();
public Node(float lat, float lon) : base(lat, lon)
{
}
public void AddConnection(Connection connection)
{
this._connections.Add(connection);
}
public Connection[] GetConnections()
{
return this._connections.ToArray();
}
}
}

View File

@ -4,7 +4,6 @@
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<RootNamespace>OSMServer</RootNamespace>
</PropertyGroup>
</Project>

View File

@ -1,21 +1,26 @@
namespace OSMImporter
namespace OSMDatastructure
{
public class Region
{
private readonly Dictionary<ulong, Node> _nodesInRegion = new Dictionary<ulong, Node>();
private readonly Dictionary<ulong, Node> _nodesInRegion = new();
public ulong regionHash { get; }
public Region(Coordinates regionCoordinates)
public Region(Coordinates regionCoordinates, float regionSize)
{
this.regionHash = regionCoordinates.GetRegionHash();
this.regionHash = regionCoordinates.GetRegionHash(regionSize);
}
public Region(ulong nodeId, Node firstNode)
public Region(ulong nodeId, Node firstNode, float regionSize)
{
this.regionHash = firstNode.GetRegionHash();
this.regionHash = firstNode.GetRegionHash(regionSize);
this._nodesInRegion.Add(nodeId, value: firstNode);
}
public Region(ulong regionHash)
{
this.regionHash = regionHash;
}
public void AddNode(ulong nodeId, Node node)
{
this._nodesInRegion.Add(nodeId, value: node);
@ -25,5 +30,6 @@ namespace OSMImporter
{
return this._nodesInRegion;
}
}
}

View File

@ -1,4 +1,4 @@
namespace OSMImporter
namespace OSMDatastructure
{
public enum highwayType : byte
{

View File

@ -1,62 +0,0 @@
namespace OSMImporter
{
public class Node : Coordinates
{
private readonly HashSet<Connection> _connections = new HashSet<Connection>();
public Node(float lat, float lon) : base(lat, lon)
{
}
public void AddConnection(Connection connection)
{
this._connections.Add(connection);
}
public Connection[] GetConnections()
{
return this._connections.ToArray();
}
/*
* Returns byte array in order:
* Value 1: Latitude (4bytes)
* Value 2: Longitude (4bytes)
* Value 3: Connections-Count (1 byte)
* Value x: Connection
*/
public byte[] ToByte()
{
long countBytes = 0;
float[] coords = { this.lat, this.lon };
countBytes += sizeof(float) * 2;
HashSet<byte[]> byteConnections = new();
foreach (Connection connection in this._connections)
{
byte[] connectionBytes = connection.ToByte();
byteConnections.Add(connectionBytes);
countBytes += connectionBytes.Length;
}
byte connectionsAmount = Convert.ToByte(byteConnections.Count);
countBytes++;
int offset = 0;
byte[] ret = new byte[countBytes];
Buffer.BlockCopy(coords, 0, ret, offset, sizeof(float) * 2);
offset += sizeof(float) * 2;
Buffer.SetByte(ret, offset, connectionsAmount);
offset++;
foreach (byte[] connection in byteConnections)
{
Buffer.BlockCopy(connection, 0, ret, offset, connection.Length);
offset += connection.Length;
}
return ret;
}
}
}

View File

@ -1,9 +1,11 @@

Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OSMImporter", "OSMImporter\OSMImporter.csproj", "{BE5C111D-AD61-433A-AE7E-F02B78551347}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OSMSplitter", "OSMSplitter\OSMSplitter.csproj", "{BE5C111D-AD61-433A-AE7E-F02B78551347}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Server", "Server\Server.csproj", "{7E025EFC-B889-4B57-A03F-EF294CFFBCD6}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OSMDatastructure", "OSMDatastructure\OSMDatastructure.csproj", "{A2489EF1-64E9-41C9-A295-B3D551D810DA}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -18,5 +20,9 @@ Global
{7E025EFC-B889-4B57-A03F-EF294CFFBCD6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7E025EFC-B889-4B57-A03F-EF294CFFBCD6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7E025EFC-B889-4B57-A03F-EF294CFFBCD6}.Release|Any CPU.Build.0 = Release|Any CPU
{A2489EF1-64E9-41C9-A295-B3D551D810DA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A2489EF1-64E9-41C9-A295-B3D551D810DA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A2489EF1-64E9-41C9-A295-B3D551D810DA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A2489EF1-64E9-41C9-A295-B3D551D810DA}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

View File

@ -0,0 +1,232 @@
using OSMDatastructure;
namespace OSMImporter;
public static class ByteConverter
{
#region Node
/*
* Node Byteform:
* +-------------------------------+
* | Latitude | Longitude | ConnectionSize| connection |
* |---------------+---------------+---------------+---------------+
* | 4 bytes | 4 bytes | 4 bytes | ConnectionSize|
* | float | float | int | Connection |
* +---------------+---------------+
*/
public static byte[] GetBytes(Node node)
{
byte[] latitudeBytes = BitConverter.GetBytes(node.lat);
byte[] longitudeBytes = BitConverter.GetBytes(node.lon);
Connection[] connections = node.GetConnections();
Dictionary<byte[], byte[]> connectionsBytes = new ();
int byteArraySize = latitudeBytes.Length + longitudeBytes.Length;
foreach (Connection connection in connections)
{
byte[] connectionBytes = ByteConverter.GetBytes(connection);
byte[] connectionSizeBytes = BitConverter.GetBytes(connectionBytes.Length);
connectionsBytes.Add(connectionSizeBytes, connectionBytes);
byteArraySize += connectionSizeBytes.Length + connectionBytes.Length;
}
byte[] retByteArray = new byte[byteArraySize];
int offset = 0;
Array.Copy(latitudeBytes, 0, retByteArray, offset, latitudeBytes.Length);
offset += latitudeBytes.Length;
Array.Copy(longitudeBytes, 0, retByteArray, offset, longitudeBytes.Length);
offset += longitudeBytes.Length;
foreach (KeyValuePair<byte[], byte[]> connectionBytes in connectionsBytes)
{
Array.Copy(connectionBytes.Key, 0, retByteArray, offset, connectionBytes.Key.Length);
offset += connectionBytes.Key.Length;
Array.Copy(connectionBytes.Value, 0, retByteArray, offset, connectionBytes.Value.Length);
offset += connectionBytes.Value.Length;
}
return retByteArray;
}
public static Node ToNode(byte[] bytes)
{
int offset = 0;
float lat = BitConverter.ToSingle(bytes, offset);
offset += sizeof(float);
float lon = BitConverter.ToSingle(bytes, offset);
offset += sizeof(float);
Node retNode = new Node(lat, lon);
Console.WriteLine("Node Lat: {0} Lon: {1}", lat, lon);
while (offset < bytes.Length)
{
int size = BitConverter.ToInt32(bytes, offset);
offset += sizeof(int);
byte[] connectionBytes = new byte[size];
Array.Copy(bytes, offset, connectionBytes, 0, size);
retNode.AddConnection(ByteConverter.ToConnection(connectionBytes));
offset += connectionBytes.Length;
}
return retNode;
}
#endregion
#region Connection
/*
* Connection Byteform:
* +---------------+---------------+
* | endId | endLatitude | endLongitude | tagBytes | tag | value |
* |---------------+---------------+---------------+---------------+---------------+---------------+
* | 8 bytes | 4 bytes | 4 bytes | 4 bytes | 1 byte | 1 byte |
* | ulong | float | float | int | | |
* +---------------+---------------+
*/
public static byte[] GetBytes(Connection connection)
{
byte[] endIdBytes = BitConverter.GetBytes(connection.endNodeId);
byte[] endLatBytes = BitConverter.GetBytes(connection.endNodeCoordinates.lat);
byte[] endLonBytes = BitConverter.GetBytes(connection.endNodeCoordinates.lon);
Dictionary<Connection.tagType, object> connectionTags = connection.GetTags();
byte[] tagBytes = BitConverter.GetBytes(connectionTags.Count * 2);
int byteArraySize = endIdBytes.Length + endLatBytes.Length + endLonBytes.Length + tagBytes.Length + connectionTags.Count * 2;
byte[] retBytes = new byte[byteArraySize];
int offset = 0;
Array.Copy(endIdBytes, 0, retBytes, offset, endIdBytes.Length);
offset += endIdBytes.Length;
Array.Copy(endLatBytes, 0, retBytes, offset, endLatBytes.Length);
offset += endLatBytes.Length;
Array.Copy(endLonBytes, 0 , retBytes, offset, endLonBytes.Length);
offset += endLonBytes.Length;
Array.Copy(tagBytes, 0, retBytes, offset, tagBytes.Length);
offset += tagBytes.Length;
foreach (KeyValuePair<Connection.tagType, object> tagKv in connectionTags)
{
retBytes[offset] = (byte)tagKv.Key;
offset++;
retBytes[offset] = (byte)tagKv.Value;
offset++;
}
return retBytes;
}
public static Connection ToConnection(byte[] bytes)
{
int offset = 0;
ulong endId = BitConverter.ToUInt64(bytes, offset);
offset += sizeof(ulong);
float endLat = BitConverter.ToSingle(bytes, offset);
offset += sizeof(float);
float endLon = BitConverter.ToSingle(bytes, offset);
offset += sizeof(float);
Connection retConnection = new Connection(endId, new Coordinates(endLat, endLon));
int tagBytes = BitConverter.ToInt32(bytes, offset);
Console.WriteLine("Connection EndId: {0} Lat: {1} Lon: {2} Tags: {3}", endId, endLat, endLon, tagBytes / 2);
offset += sizeof(int);
while (offset < bytes.Length)
{
byte key = bytes[offset];
byte value = bytes[offset + 1];
retConnection.AddTag(key, value);
offset += 2;
}
return retConnection;
}
#endregion
#region Region
/*
* Region Byteform:
* +---------------+---------------+---------------+
* | regionHash | nodeBytes | nodeId | node |
* |---------------+---------------+---------------+---------------+
* | 8 bytes | 4 bytes | 8 bytes | nodeBytes |
* | ulong | int | ulong | Node |
* +---------------+---------------+---------------+
*/
public static byte[] GetBytes(Region region)
{
byte[] regionHashBytes = BitConverter.GetBytes(region.regionHash);
int byteArraySize = regionHashBytes.Length;
Dictionary<byte[], byte[]> nodesBytes = new();
foreach(KeyValuePair<ulong, Node> nodeKv in region.GetNodes())
{
byte[] nodeIdBytes = BitConverter.GetBytes(nodeKv.Key);
byte[] nodeBytes = GetBytes(nodeKv.Value);
nodesBytes.Add(nodeIdBytes, nodeBytes);
byteArraySize += sizeof(int) + nodeIdBytes.Length + nodeBytes.Length;
}
byte[] retBytes = new byte[byteArraySize];
int offset = 0;
Array.Copy(regionHashBytes, 0, retBytes, offset, regionHashBytes.Length);
offset += regionHashBytes.Length;
foreach (KeyValuePair<byte[], byte[]> nodeByteKv in nodesBytes)
{
Array.Copy(BitConverter.GetBytes(nodeByteKv.Value.Length), 0, retBytes, offset, sizeof(int));
offset += sizeof(int);
Array.Copy(nodeByteKv.Key, 0, retBytes, offset, nodeByteKv.Key.Length);
offset += nodeByteKv.Key.Length;
Array.Copy(nodeByteKv.Value, 0, retBytes, offset, nodeByteKv.Value.Length);
offset += nodeByteKv.Value.Length;
}
return retBytes;
}
public static Region ToRegion(byte[] bytes)
{
int offset = 0;
ulong regionHash = BitConverter.ToUInt64(bytes, offset);
offset += sizeof(ulong);
Console.WriteLine("Region: {0}", regionHash);
Region retRegion = new Region(regionHash);
while (offset < bytes.Length)
{
int nodeBytesAmount = BitConverter.ToInt32(bytes, offset);
offset += sizeof(int);
ulong nodeId = BitConverter.ToUInt64(bytes, offset);
offset += sizeof(ulong);
byte[] nodeBytes = new byte[nodeBytesAmount];
Array.Copy(bytes, offset, nodeBytes, 0, nodeBytesAmount);
retRegion.AddNode(nodeId, ByteConverter.ToNode(nodeBytes));
offset += nodeBytes.Length;
}
return retRegion;
}
#endregion
}

View File

@ -1,6 +1,7 @@
using System.Globalization;
using System.Text;
using System.Xml;
using OSMDatastructure;
namespace OSMImporter
{
@ -19,68 +20,6 @@ namespace OSMImporter
NumberDecimalSeparator = "."
};
public static Region ImportRegion(string folderPath, Coordinates coordinates)
{
string fullPath = Path.Combine(folderPath, coordinates.GetRegionHash().ToString());
Console.WriteLine(fullPath);
Region retRegion = new Region(coordinates);
if (!File.Exists(fullPath))
return retRegion;
FileStream fileStream = new FileStream(fullPath, FileMode.Open);
while (fileStream.Position < fileStream.Length)
{
byte[] nodeIdBytes = new byte[sizeof(ulong)];
fileStream.ReadExactly(nodeIdBytes,0,nodeIdBytes.Length);
ulong[] nodeId = new ulong[1];
Buffer.BlockCopy(nodeIdBytes,0,nodeId,0,nodeIdBytes.Length);
byte[] latBytes = new byte[sizeof(float)];
fileStream.ReadExactly(latBytes, 0, latBytes.Length);
byte[] lonBytes = new byte[sizeof(float)];
fileStream.ReadExactly(lonBytes, 0, latBytes.Length);
Node newNode = NodeFromBytes(latBytes, lonBytes);
retRegion.AddNode(nodeId[0], newNode);
byte connectionsAmount = Convert.ToByte(fileStream.ReadByte());
for (byte connectionInt = 0; connectionInt < connectionsAmount; connectionInt++)
{
byte[] endIdBytes = new byte[sizeof(ulong)];
fileStream.ReadExactly(endIdBytes, 0, endIdBytes.Length);
Connection connection = ConnectionFromByte(endIdBytes);
newNode.AddConnection(connection);
byte tagsAmount = Convert.ToByte(fileStream.ReadByte());
for (byte tagInt = 0; tagInt < tagsAmount; tagInt++)
{
byte keyByte = Convert.ToByte(fileStream.ReadByte());
byte valueByte = Convert.ToByte(fileStream.ReadByte());
connection.AddTag(keyByte, valueByte);
}
}
}
fileStream.Close();
return retRegion;
}
private static Node NodeFromBytes(byte[] latByte, byte[] lonByte)
{
float[] lat = new float[1];
float[] lon = new float[1];
Buffer.BlockCopy(latByte, 0, lat, 0, latByte.Length);
Buffer.BlockCopy(lonByte, 0, lon, 0, lonByte.Length);
return new Node(lat[0], lon[0]);
}
private static Connection ConnectionFromByte(byte[] endIdByte)
{
ulong[] endId = new ulong[1];
Buffer.BlockCopy(endIdByte, 0, endId, 0, endIdByte.Length);
return new Connection(endId[0]);
}
public static void Split(string xmlFilePath, string outputFolderPath)
{
if (!File.Exists(xmlFilePath))
@ -213,27 +152,23 @@ namespace OSMImporter
if (oneWay)
{
if(nodes.ContainsKey(nodeIds[i]))
nodes[nodeIds[i]]!.AddConnection(new Connection(nodeIds[i + 1], tags));
nodes[nodeIds[i]]!.AddConnection(new Connection(nodeIds[i + 1], nodes[nodeIds[i + 1]]!, tags));
if(nodes.ContainsKey(nodeIds[i + 1]))
nodes[nodeIds[i + 1]]!.AddConnection(new Connection(nodeIds[i], tags));
nodes[nodeIds[i + 1]]!.AddConnection(new Connection(nodeIds[i], nodes[nodeIds[i]]!, tags));
}
else if (forward)
{
if(nodes.ContainsKey(nodeIds[i]))
nodes[nodeIds[i]]!.AddConnection(new Connection(nodeIds[i + 1], tags));
nodes[nodeIds[i]]!.AddConnection(new Connection(nodeIds[i + 1], nodes[nodeIds[i + 1]]!, tags));
}
else
{
if(nodes.ContainsKey(nodeIds[i + 1]))
nodes[nodeIds[i + 1]]!.AddConnection(new Connection(nodeIds[i], tags));
nodes[nodeIds[i + 1]]!.AddConnection(new Connection(nodeIds[i], nodes[nodeIds[i]]!, tags));
}
}
}
/*
* Value x: nodeId (4bytes)
* Value x+1: Node (xbytes)
*/
private static void WriteRegion(Region region, string outputFolderPath)
{
if (!Directory.Exists(outputFolderPath))
@ -243,16 +178,7 @@ namespace OSMImporter
if (!File.Exists(fullPath))
File.Create(fullPath).Close();
FileStream fileStream = new FileStream(fullPath, FileMode.Append);
foreach (KeyValuePair<ulong, Node> nodeKeyValuePair in region.GetNodes())
{
byte[] nodeByte = nodeKeyValuePair.Value.ToByte();
byte[] nodeIdByte = new byte[sizeof(long)];
ulong[] nodeId = new ulong[]{nodeKeyValuePair.Key};
Buffer.BlockCopy(nodeId,0,nodeIdByte,0,nodeIdByte.Length);
fileStream.Write(nodeIdByte);
fileStream.Write(nodeByte, 0, nodeByte.Length );
}
fileStream.Write(ByteConverter.GetBytes(region));
fileStream.Close();
}
}

View File

@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<RootNamespace>OSMServer</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OSMDatastructure\OSMDatastructure.csproj" />
</ItemGroup>
</Project>

View File

@ -1,3 +1,5 @@
using OSMDatastructure;
namespace OSMImporter
{
public class RegionManager
@ -11,11 +13,11 @@ namespace OSMImporter
public Region GetRegion(Coordinates coordinates)
{
if(this._regions.ContainsKey(coordinates.GetRegionHash()))
return this._regions[coordinates.GetRegionHash()];
if(this._regions.ContainsKey(coordinates.GetRegionHash(Importer.regionSize)))
return this._regions[coordinates.GetRegionHash(Importer.regionSize)];
else
{
Region newRegion = new Region(coordinates);
Region newRegion = new Region(coordinates, Importer.regionSize);
this._regions.Add(newRegion.regionHash, value: newRegion);
return newRegion;
}

View File

@ -1,16 +1,31 @@
using OSMImporter;
using OSMDatastructure;
namespace Server;
public class Server
{
public static Region LoadRegion(string folderPath, Coordinates coordinates)
{
string fullPath = Path.Combine(folderPath, coordinates.GetRegionHash(Importer.regionSize).ToString());
Console.WriteLine(fullPath);
Region retRegion = new Region(coordinates, Importer.regionSize);
if (!File.Exists(fullPath))
return retRegion;
FileStream fileStream = new FileStream(fullPath, FileMode.Open);
byte[] regionBytes = new byte[fileStream.Length];
fileStream.Read(regionBytes, 0, regionBytes.Length);
fileStream.Close();
return ByteConverter.ToRegion(regionBytes);
}
public static void Main(string[] args)
{
Importer.Split("/home/glax/Downloads/bayern-latest.osm", "/home/glax/Downloads/bayern-latest");
Region r = Importer.ImportRegion("/home/glax/Downloads/bayern-latest", new Coordinates(47.890f,12.56f));
foreach(KeyValuePair<ulong, Node> nodes in r.GetNodes())
Console.WriteLine(nodes.Key);
Region r = LoadRegion("/home/glax/Downloads/bayern-latest", new Coordinates(47.890f,12.56f));
}
}

View File

@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OSMImporter\OSMImporter.csproj" />
<ProjectReference Include="..\OSMSplitter\OSMSplitter.csproj" />
</ItemGroup>
</Project>