glax/OSMServer
1
0
Fork 0
Code Pull Requests Releases Wiki Activity

Combining Datastructures and Serialization

Browse Source 
//TODO
This commit is contained in:
glax 2023-03-30 18:24:57 +02:00
parent 7560d59c25
commit ac1ac62a00
17 changed files with 90 additions and 1350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

272
OSMDatastructure/ByteConverter.cs
View File

@ -1,272 +0,0 @@
using System.Text;
using OSMDatastructure.Graph;
namespace OSMDatastructure;
public static class ByteConverter
{
#region Region
/*
* | regionHash | Nodes |
* |---------------+---------------+
* | 4 bytes |
* | int |
*/
public static byte[] GetBytes(Region region)
{
int totalNodeBytes = 0;
HashSet<byte[]> nodes = new();
foreach (OsmNode node in region.nodes)
{
byte[] nodeBytes = GetBytes(node);
totalNodeBytes += nodeBytes.Length;
nodes.Add(nodeBytes);
}
byte[] retBytes = new byte[sizeof(int) + totalNodeBytes];
Array.Copy(BitConverter.GetBytes(region.regionHash), 0, retBytes, 0, sizeof(int));
int offset = sizeof(int);
foreach (byte[] node in nodes)
{
Array.Copy(node, 0, retBytes, offset, node.Length);
offset += node.Length;
}
return retBytes;
}
public static Region ToRegion(byte[] bytes)
{
Region retRegion = new Region(BitConverter.ToInt32(bytes, 0));
int offset = sizeof(int);
while (offset < bytes.Length)
{
int size = BitConverter.ToInt32(bytes, offset);
byte[] nodeBytes = new byte[size];
Array.Copy(bytes, offset, nodeBytes, 0, size);
offset += size;
retRegion.nodes.Add(ToNode(nodeBytes));
}
return retRegion;
}
#endregion
#region Coordinates
/*
* | Latitude | Longitude |
* |---------------+---------------+
* | 4 bytes | 4 bytes |
* | float | float |
*/
public static byte[] GetBytes(Coordinates coordinates)
{
byte[] retBytes = new byte[8];
Array.Copy(BitConverter.GetBytes(coordinates.latitude), 0, retBytes, 0, sizeof(float));
Array.Copy(BitConverter.GetBytes(coordinates.longitude), 0, retBytes, sizeof(float), sizeof(float));
return retBytes;
}
public static Coordinates ToCoordinates(byte[] bytes)
{
if (bytes.Length != 8)
throw new ArgumentException("Needs exactly 8 bytes", nameof(bytes));
float latitude = BitConverter.ToSingle(bytes, 0);
float longitude = BitConverter.ToSingle(bytes, sizeof(float));
return new Coordinates(latitude, longitude);
}
#endregion
#region OsmNode
/*
* | Size | Coordinates | Edges
* |---------------+---------------+---------------+
* | 4 bytes | 8 bytes |
* | int | byte[] |
*/
public static byte[] GetBytes(OsmNode node)
{
int totalBytes = sizeof(int) + 8; //size + Coordinates
HashSet<byte[]> edges = new();
foreach (OsmEdge edge in node.edges)
{
byte[] edgeBytes = GetBytes(edge);
edges.Add(edgeBytes);
totalBytes += edgeBytes.Length;
}
byte[] retBytes = new byte[totalBytes];
Array.Copy(BitConverter.GetBytes(totalBytes), retBytes, 4);
Array.Copy(GetBytes(node.coordinates), 0, retBytes, 4, 8);
int offset = 4 + 8;
foreach (byte[] edgeBytes in edges)
{
Array.Copy(edgeBytes, 0, retBytes, offset, edgeBytes.Length);
offset += edgeBytes.Length;
}
return retBytes;
}
public static OsmNode ToNode(byte[] bytes)
{
byte[] coordinateBytes = new byte[8];
Array.Copy(bytes, 4, coordinateBytes, 0, 8);
Coordinates coordinates = ToCoordinates(coordinateBytes);
OsmNode retNode = new OsmNode(coordinates);
int offset = sizeof(int) + 8;
while (offset < bytes.Length)
{
int tagsSize = BitConverter.ToInt32(bytes, offset);
byte[] edgeBytes = new byte[sizeof(int) + 8 + tagsSize];
Array.Copy(bytes, offset, edgeBytes, 0, edgeBytes.Length);
offset += edgeBytes.Length;
retNode.edges.Add(ToEdge(edgeBytes));
}
return retNode;
}
#endregion
#region OsmEdge
/* Edge:
* | tagsSize | neighborCoord | tags
* |---------------+---------------+--------------
* | 4 bytes | 8 bytes |
* | int | byte[] |
*
*/
public static byte[] GetBytes(OsmEdge edge)
{
int totalBytes = sizeof(int) + 8; //size + Coordinates
int offset = totalBytes;
int tagsSize = 0;
HashSet<byte[]> tags = new();
foreach (KeyValuePair<OsmEdge.tagType, object> kv in edge.tags)
{
byte[] tagBytes = GetBytes(kv);
tags.Add(tagBytes);
tagsSize += tagBytes.Length;
}
totalBytes += tagsSize;
byte[] retBytes = new byte[totalBytes];
Array.Copy(BitConverter.GetBytes(tagsSize), retBytes, 4);
Array.Copy(GetBytes(edge.neighborCoordinates), 0, retBytes, sizeof(int), 8);
foreach (byte[] tag in tags)
{
Array.Copy(tag, 0, retBytes, offset, tag.Length);
offset += tag.Length;
}
return retBytes;
}
public static OsmEdge ToEdge(byte[] bytes)
{
byte[] coordinateBytes = new byte[8];
Array.Copy(bytes, 4, coordinateBytes, 0, 8);
Coordinates coordinates = ToCoordinates(coordinateBytes);
OsmEdge retEdge = new OsmEdge(coordinates);
int offset = sizeof(int) + 8;
while (offset < bytes.Length)
{
int tagContentSize = BitConverter.ToInt32(bytes, offset);
byte[] tagBytes = new byte[sizeof(int) + 1 + tagContentSize];
Array.Copy(bytes, offset, tagBytes, 0, tagBytes.Length);
offset += tagBytes.Length;
KeyValuePair<OsmEdge.tagType, object> tag = ToTag(tagBytes);
retEdge.tags.Add(tag.Key, tag.Value);
}
return retEdge;
}
#endregion
#region OsmEdge.Tag
/*
* Tag:
* | contentSize | tagType | tagContent
* |---------------+---------------+---------------
* | 4 bytes | 1 byte |
* | int | byte |
*/
public static byte[] GetBytes(KeyValuePair<OsmEdge.tagType, object> tag)
{
byte[] objectBytes;
Type objectType = tag.Value.GetType();
if (objectType == Type.GetType("System.String"))
{
objectBytes = Encoding.ASCII.GetBytes((string)tag.Value);
}
else if (objectType == Type.GetType("System.Boolean"))
{
objectBytes = BitConverter.GetBytes((bool)tag.Value);
}
else if (objectType == Type.GetType("System.Int32"))
{
objectBytes = BitConverter.GetBytes((int)tag.Value);
}
else if (objectType == Type.GetType("System.UInt64"))
{
objectBytes = BitConverter.GetBytes((ulong)tag.Value);
}
else if (objectType == Type.GetType("System.Byte") || objectType.Name == "wayType")
{
objectBytes = new[] { (byte)tag.Value };
}
else
{
throw new Exception(string.Format("Tag-Bytes object-Type: {0}", tag.Value.GetType()));
}
byte[] retBytes = new byte[sizeof(int) + 1 + objectBytes.Length];
Array.Copy(BitConverter.GetBytes(objectBytes.Length), 0, retBytes, 0, sizeof(int));
retBytes[sizeof(int)] = (byte)tag.Key;
Array.Copy(objectBytes, 0, retBytes, sizeof(int) + 1, objectBytes.Length);
return retBytes;
}
public static KeyValuePair<OsmEdge.tagType, object> ToTag(byte[] bytes)
{
OsmEdge.tagType tagType = (OsmEdge.tagType)bytes[sizeof(int)];
int contentSize = BitConverter.ToInt32(bytes, 0);
byte[] content = new byte[contentSize];
Array.Copy(bytes, sizeof(int) + 1, content, 0, contentSize);
switch (tagType)
{
case OsmEdge.tagType.highway:
return new KeyValuePair<OsmEdge.tagType, object>(tagType, (OsmEdge.wayType)content[0]);
case OsmEdge.tagType.maxspeed:
return new KeyValuePair<OsmEdge.tagType, object>(tagType, content[0]);
case OsmEdge.tagType.oneway:
return new KeyValuePair<OsmEdge.tagType, object>(tagType, BitConverter.ToBoolean(content));
case OsmEdge.tagType.forward:
return new KeyValuePair<OsmEdge.tagType, object>(tagType, BitConverter.ToBoolean(content));
case OsmEdge.tagType.id:
return new KeyValuePair<OsmEdge.tagType, object>(tagType, BitConverter.ToUInt64(content));
default:
return new KeyValuePair<OsmEdge.tagType, object>(tagType, content);
}
}
#endregion
}

30
OSMDatastructure/Coordinates.cs
View File

@ -1,5 +1,8 @@
using System.Globalization;
namespace OSMDatastructure.Graph;
[Serializable]
public class Coordinates
{
public float latitude { get; }
@ -20,37 +23,32 @@ public class Coordinates
return convObj.latitude == this.latitude && convObj.longitude == this.longitude;
}
private const float decimalCoordsSave = 10000;
private const ulong offset = 10000000;
//Latitude maxChars = 7
//Longitude maxChars = 8
public override int GetHashCode()
{
return GetHashCode(latitude, longitude);
}
public static int GetRegionHashCode(float latitude, float longitude)
public static ulong GetRegionHashCode(float latitude, float longitude)
{
float latRegion = latitude - latitude % Region.RegionSize;
float lonRegion = longitude - longitude % Region.RegionSize;
return GetHashCode(latRegion, lonRegion);
}
public static int GetRegionHashCode(Coordinates coordinates)
public static ulong GetRegionHashCode(Coordinates coordinates)
{
float latRegion = coordinates.latitude - coordinates.latitude % Region.RegionSize;
float lonRegion = coordinates.longitude - coordinates.longitude % Region.RegionSize;
return GetHashCode(latRegion, lonRegion);
}
public static int GetHashCode(float latitude, float longitude)
private const float decimalCoordsSave = 10000; //Latitude maxChars = 7
private const ulong offset = 10000000; //Longitude maxChars = 8
public static ulong GetHashCode(float latitude, float longitude)
{
return HashCode.Combine(latitude, longitude);
ulong latHash = Convert.ToUInt64((latitude + 90) * decimalCoordsSave);
ulong lonHash = Convert.ToUInt64((longitude + 180) * decimalCoordsSave);
return latHash * offset + lonHash;
}
public override string ToString()
{
return string.Format("COORDINATES Lat: {0} Lon: {1}", this.latitude, this.longitude);
return
$"COORDINATES Lat: {this.latitude.ToString(NumberFormatInfo.InvariantInfo)} Lon: {this.longitude.ToString(CultureInfo.InvariantCulture)}";
}
}

226
OSMDatastructure/OSMEdge.cs
View File

@ -1,228 +1,22 @@
namespace OSMDatastructure.Graph;
[Serializable]
public class OsmEdge
{
public Coordinates neighborCoordinates { get; }
public readonly Dictionary<tagType, object> tags = new();
public OsmEdge(float lat, float lon)
{
this.neighborCoordinates = new Coordinates(lat, lon);
this.tags = new();
}
public ulong wayId { get; }
public ulong neighborId { get; }
public ulong neighborRegion { get; }
public OsmEdge(Coordinates neighborCoordinates)
public OsmEdge(ulong wayID, ulong neighborID, ulong neighborRegion)
{
this.neighborCoordinates = neighborCoordinates;
this.tags = new();
}
public OsmEdge(float lat, float lon, Dictionary<tagType, object> tags)
{
this.neighborCoordinates = new Coordinates(lat, lon);
//To prevent "EMPTY" tags
foreach (KeyValuePair<tagType, object> tag in tags)
{
if(tag.Key != tagType.EMPTY)
this.tags.Add(tag.Key, tag.Value);
}
}
public OsmEdge(Coordinates neighborCoordinates, Dictionary<tagType, object> tags)
{
this.neighborCoordinates = neighborCoordinates;
//To prevent "EMPTY" tags
foreach (KeyValuePair<tagType, object> tag in tags)
{
if(tag.Key != tagType.EMPTY)
this.tags.Add(tag.Key, tag.Value);
}
}
public enum tagType : byte
{
highway, oneway, footway, sidewalk, cycleway, busway, forward, maxspeed, name, surface, lanes, access, tracktype, id, EMPTY
}
private static readonly Dictionary<wayType, byte> defaultSpeedCar = new() {
{ wayType.NONE, 0 },
{ wayType.motorway, 110 },
{ wayType.trunk, 100 },
{ wayType.primary, 80 },
{ wayType.secondary, 80 },
{ wayType.tertiary, 70 },
{ wayType.unclassified, 20 },
{ wayType.residential, 10 },
{ wayType.motorway_link, 50 },
{ wayType.trunk_link, 50 },
{ wayType.primary_link, 30 },
{ wayType.secondary_link, 25 },
{ wayType.tertiary_link, 25 },
{ wayType.living_street, 10 },
{ wayType.service, 0 },
{ wayType.pedestrian, 0 },
{ wayType.track, 0 },
{ wayType.bus_guideway, 0 },
{ wayType.escape, 0 },
{ wayType.raceway, 0 },
{ wayType.road, 25 },
{ wayType.busway, 0 },
{ wayType.footway, 0 },
{ wayType.bridleway, 0 },
{ wayType.steps, 0 },
{ wayType.corridor, 0 },
{ wayType.path, 0 },
{ wayType.cycleway, 0 },
{ wayType.construction, 0 }
};
private static readonly Dictionary<wayType, byte> defaultSpeedPedestrian = new() {
{ wayType.NONE, 0 },
{ wayType.motorway, 0 },
{ wayType.trunk, 0 },
{ wayType.primary, 0 },
{ wayType.secondary, 0 },
{ wayType.tertiary, 0 },
{ wayType.unclassified, 1 },
{ wayType.residential, 3 },
{ wayType.motorway_link, 0 },
{ wayType.trunk_link, 0 },
{ wayType.primary_link, 0 },
{ wayType.secondary_link, 0 },
{ wayType.tertiary_link, 0 },
{ wayType.living_street, 5 },
{ wayType.service, 2 },
{ wayType.pedestrian, 5 },
{ wayType.track, 0 },
{ wayType.bus_guideway, 0 },
{ wayType.escape, 0 },
{ wayType.raceway, 0 },
{ wayType.road, 3 },
{ wayType.busway, 0 },
{ wayType.footway, 4 },
{ wayType.bridleway, 1 },
{ wayType.steps, 2 },
{ wayType.corridor, 3 },
{ wayType.path, 4 },
{ wayType.cycleway, 2 },
{ wayType.construction, 0 }
};
public enum wayType : byte { NONE, motorway, trunk, primary, secondary, tertiary, unclassified, residential, motorway_link, trunk_link, primary_link, secondary_link, tertiary_link, living_street, service, pedestrian, track, bus_guideway, escape, raceway, road, busway, footway, bridleway, steps, corridor, path, cycleway, construction }
public enum speedType { pedestrian, car, road }
public void AddTag(string key, string value)
{
KeyValuePair<tagType, object> tag = ConvertToTag(key, value);
if(tag.Key != tagType.EMPTY)
tags.Add(tag.Key, tag.Value);
}
public static KeyValuePair<tagType, object> ConvertToTag(string key, string value)
{
switch (key)
{
case "highway":
try
{
return new KeyValuePair<tagType, object>(tagType.highway, (wayType)Enum.Parse(typeof(wayType), value, true));
}
catch (ArgumentException)
{
return new KeyValuePair<tagType, object>(tagType.highway, wayType.unclassified);
}
case "maxspeed":
try
{
byte speed = Convert.ToByte(value);
if(speed == 255)
return new KeyValuePair<tagType, object>(tagType.EMPTY, 0);
else
return new KeyValuePair<tagType, object>(tagType.maxspeed, speed);
}
catch (Exception)
{
//Console.WriteLine(e);
//Console.WriteLine("Continuing...");
return new KeyValuePair<tagType, object>(tagType.maxspeed, byte.MaxValue);
}
case "oneway":
switch (value)
{
case "yes":
return new KeyValuePair<tagType, object>(tagType.oneway, true);
case "-1":
return new KeyValuePair<tagType, object>(tagType.forward, false);
case "no":
return new KeyValuePair<tagType, object>(tagType.oneway, false);
}
break;
case "id":
return new KeyValuePair<tagType, object>(tagType.id, Convert.ToUInt64(value));
}
return new KeyValuePair<tagType, object>(tagType.EMPTY, 0);
}
public ulong GetId()
{
return tags.ContainsKey(tagType.id) ? (ulong)tags[tagType.id] : 0;
}
public wayType GetHighwayType()
{
if (!tags.ContainsKey(tagType.highway))
throw new Exception("Not a road?");
return (wayType)tags[tagType.highway];
}
public bool IsOneWay()
{
return tags.ContainsKey(tagType.oneway) && (bool)tags[tagType.oneway];
}
public byte GetMaxSpeed(speedType type)
{
switch (type)
{
case speedType.road:
return tags.ContainsKey(tagType.maxspeed) ? (byte)tags[tagType.maxspeed] : (byte)0;
case speedType.car:
return tags.ContainsKey(tagType.maxspeed) ? (byte)tags[tagType.maxspeed] : defaultSpeedCar[GetHighwayType()];
case speedType.pedestrian:
return defaultSpeedPedestrian[GetHighwayType()];
default:
return 0;
}
}
public bool IsForward()
{
return this.tags.ContainsKey(tagType.forward) && (bool)this.tags[tagType.forward];
}
public double GetWeight(OsmNode parentNode, speedType vehicle)
{
double distance = Utils.DistanceBetween(parentNode, neighborCoordinates);
byte speedByte = GetMaxSpeed(vehicle);
if (speedByte > 0)
{
double speed = Convert.ToDouble(speedByte);
return distance / speed;
}
else
{
return double.MaxValue;
}
this.wayId = wayID;
this.neighborId = neighborID;
this.neighborRegion = neighborRegion;
}
public override string ToString()
{
string tagsString = "";
foreach (KeyValuePair<tagType, object> tag in tags)
{
tagsString += string.Format("\n\t{0}: {1}", tag.Key, tag.Value);
}
return string.Format("EDGE neighborCoordinates: {0} {1}", neighborCoordinates.ToString(), tagsString);
return $"EDGE WayID: {wayId} NeighborID: {neighborId} {neighborRegion}";
}
}

20
OSMDatastructure/OsmNode.cs
View File

@ -1,23 +1,27 @@
namespace OSMDatastructure.Graph;
[Serializable]
public class OsmNode
{
public ulong nodeId { get; }
public HashSet<OsmEdge> edges { get; }
public Coordinates coordinates { get; }
public OsmNode? previousPathNode = null;
public double currentPathWeight = double.MaxValue;
public double currentPathLength = double.MaxValue;
public double directDistanceToGoal = double.MaxValue;
[NonSerialized]public OsmNode? previousPathNode = null;
[NonSerialized]public double currentPathWeight = double.MaxValue;
[NonSerialized]public double currentPathLength = double.MaxValue;
[NonSerialized]public double directDistanceToGoal = double.MaxValue;
public OsmNode(float lat, float lon)
public OsmNode(ulong nodeId, float lat, float lon)
{
this.nodeId = nodeId;
this.edges = new();
this.coordinates = new Coordinates(lat, lon);
}
public OsmNode(Coordinates coordinates)
public OsmNode(ulong nodeId, Coordinates coordinates)
{
this.nodeId = nodeId;
this.edges = new();
this.coordinates = coordinates;
}
@ -25,14 +29,14 @@ public class OsmNode
public OsmEdge? GetEdgeToNode(OsmNode n)
{
foreach (OsmEdge e in this.edges)
if (e.neighborCoordinates.Equals(n.coordinates))
if (e.neighborId == n.nodeId)
return e;
return null;
}
public override bool Equals(object? obj)
{
return obj != null && obj.GetType() == this.GetType() && ((OsmNode)obj).coordinates.Equals(this.coordinates);
return obj != null && obj.GetType() == this.GetType() && ((OsmNode)obj).nodeId == this.nodeId;
}
public override int GetHashCode()

10
OSMDatastructure/Region.cs
View File

@ -2,20 +2,24 @@ using OSMDatastructure.Graph;
namespace OSMDatastructure;
[Serializable]
public class Region
{
public const float RegionSize = 0.01f;
[NonSerialized]public const float RegionSize = 0.01f;
public readonly HashSet<OsmNode> nodes = new();
public int regionHash { get; }
public ulong regionHash { get; }
public TagManager tagManager { get; }
public Region(int regionHash)
public Region(ulong regionHash)
{
this.regionHash = regionHash;
tagManager = new TagManager();
}
public Region(Coordinates coordinates)
{
regionHash = Coordinates.GetRegionHashCode(coordinates);
tagManager = new TagManager();
}
public OsmNode? GetNodeWithCoordinates(Coordinates coordinates)

3
OsmXmlToRegionConverter/Tag.cs → OSMDatastructure/Tag.cs
View File

@ -1,5 +1,6 @@
namespace OsmXmlToRegionConverter;
namespace OSMDatastructure.Graph;
[Serializable]
public class Tag
{
public TagType key { get; }

45
OSMDatastructure/TagManager.cs Normal file
View File

@ -0,0 +1,45 @@
using System.Runtime.CompilerServices;
namespace OSMDatastructure.Graph;
[Serializable]
public class TagManager
{
public readonly Dictionary<ulong, HashSet<Tag>> wayTags = new();
public bool ContainsKey(ulong wayId, Tag.TagType key)
{
return wayTags.ContainsKey(wayId) && wayTags[wayId].Any(tag => tag.key == key);
}
public object? GetTag(ulong wayId, Tag.TagType key)
{
return ContainsKey(wayId, key) ? wayTags[wayId].First(tag => tag.key == key) : null;
}
public void AddTag(ulong wayId, string key, string value)
{
Tag tag = Tag.ConvertToTag(key, value);
AddTag(wayId, tag);
}
public void AddTag(ulong wayId, Tag tag)
{
if (tag.key != Tag.TagType.EMPTY)
{
if(!wayTags.ContainsKey(wayId))
wayTags.Add(wayId, new HashSet<Tag>());
wayTags[wayId].Add(tag);
}
}
public HashSet<Tag>? GetTagsForWayId(ulong wayId)
{
return wayTags.TryGetValue(wayId, out HashSet<Tag>? value) ? value : null;
}
public bool ContainsWay(ulong wayId)
{
return wayTags.ContainsKey(wayId);
}
}

259
OsmXmlToRegionConverter/ByteConverter.cs
View File

@ -1,259 +0,0 @@
using System.Text;
namespace OsmXmlToRegionConverter;
public static class Extensions
{
public static T[] SubArray<T>(this T[] array, int offset, int length)
{
T[] result = new T[length];
Buffer.BlockCopy(array, offset, result, 0, length);
return result;
}
}
public class ByteConverter
{
private enum ContentType : byte
{
Node = 0,
Coordinates = 1,
Edge = 2,
TmpNode = 3,
TmpWay = 4,
Region = 5,
TagManager = 6
}
public static HashSet<object> GetObjectsFromBytes(byte[] bytes)
{
HashSet<object> ret = new HashSet<object>();
while (bytes.Length > 0)
{
ContentType contentType = (ContentType)bytes[0];
switch (contentType)
{
case ContentType.Node:
int edgeCount = BitConverter.ToInt32(bytes, 1);
int byteSizeNode = Node.ByteSizeEmpty + 1 + edgeCount * Edge.ByteSize; //+1 for contenttype of coordinates
ret.Add(Node.FromBytes(bytes.SubArray(1, byteSizeNode)));
bytes = bytes.SubArray(1 + byteSizeNode, bytes.Length - 1 - byteSizeNode); //-1 for overflow
break;
case ContentType.Coordinates:
ret.Add(Coordinates.FromBytes(bytes.SubArray(1, Coordinates.ByteSize)));
bytes = bytes.SubArray(1 + Coordinates.ByteSize, bytes.Length - 1 - Coordinates.ByteSize);
break;
case ContentType.Edge:
ret.Add(Edge.FromBytes(bytes.SubArray(1, Edge.ByteSize)));
bytes = bytes.SubArray(1 + Edge.ByteSize, bytes.Length - 1 - Edge.ByteSize);
break;
case ContentType.TmpNode:
ret.Add(TmpNode.FromBytes(bytes.SubArray(1, TmpNode.ByteSize + 1))); //+1 for contenttype of coordinates
bytes = bytes.SubArray(1 + TmpNode.ByteSize + 1, bytes.Length - 1 - (TmpNode.ByteSize + 1));
break;
case ContentType.TmpWay:
int byteSizeTmpWay = BitConverter.ToInt32(bytes, 1);
ret.Add(TmpWay.FromBytes(bytes.SubArray(1, byteSizeTmpWay)));
bytes = bytes.SubArray(1 + byteSizeTmpWay, bytes.Length - 1 - byteSizeTmpWay);
break;
case ContentType.TagManager:
int totalBytesTagManager = BitConverter.ToInt32(bytes, 1);
ret.Add(TagManager.FromBytes(bytes.SubArray(1 + sizeof(int), totalBytesTagManager)));
bytes = bytes.SubArray(1 + sizeof(int) + totalBytesTagManager, bytes.Length - 1 - sizeof(int) - totalBytesTagManager);
break;
default:
throw new Exception("Unknown Type");
}
}
return ret;
}
public static byte[] GetBytes(Node n)
{
using (MemoryStream m = new MemoryStream())
{
using (BinaryWriter b = new BinaryWriter(m))
{
b.Write((byte)ContentType.Node);
b.Write(n.edges.Count);
b.Write((byte)ContentType.Node);
b.Write(n.id);
b.Write(GetBytes(n.coordinates));
foreach(Edge e in n.edges)
b.Write(GetBytes(e));
}
return m.ToArray();
}
}
public static byte[] GetBytes(Coordinates c)
{
using (MemoryStream m = new MemoryStream())
{
using (BinaryWriter b = new BinaryWriter(m))
{
b.Write((byte)ContentType.Coordinates);
b.Write(c.latitude);
b.Write(c.longitude);
}
return m.ToArray();
}
}
public static byte[] GetBytes(Edge e)
{
using (MemoryStream m = new MemoryStream())
{
using (BinaryWriter b = new BinaryWriter(m))
{
b.Write((byte)ContentType.Edge);
b.Write(e.wayId);
b.Write(e.neighborId);
b.Write(e.neighborRegion);
}
return m.ToArray();
}
}
public static byte[] GetBytes(TmpNode tn)
{
using (MemoryStream m = new MemoryStream())
{
using (BinaryWriter b = new BinaryWriter(m))
{
b.Write((byte)ContentType.TmpNode);
b.Write(tn.id);
b.Write(GetBytes(tn.coordinates));
}
return m.ToArray();
}
}
public static byte[] GetBytes(TmpWay tw)
{
using (MemoryStream m = new MemoryStream())
{
using (BinaryWriter b = new BinaryWriter(m))
{
b.Write((byte)ContentType.TmpWay);
HashSet<byte[]> tagBytesSet = new HashSet<byte[]>();
int totalTagBytes = 0;
foreach (Tag tag in tw.tags)
{
byte[] tagBytes = GetBytes(tag);
totalTagBytes += tagBytes.Length;
tagBytesSet.Add(tagBytes);
}
b.Write(sizeof(ulong) + sizeof(int) + tw.wayNodeIds.Length * sizeof(ulong) + totalTagBytes);
b.Write(tw.id);
b.Write(tw.wayNodeIds.Length);
foreach (ulong wayNodeId in tw.wayNodeIds)
b.Write(wayNodeId);
foreach (byte[] tagBytes in tagBytesSet)
b.Write(tagBytes);
}
return m.ToArray();
}
}
public static byte[] GetBytes(Region r)
{
using (MemoryStream m = new MemoryStream())
{
using (BinaryWriter b = new BinaryWriter(m))
{
b.Write((byte)ContentType.Region);
b.Write(r.GetNodes().Length);
b.Write(r.regionHash);
foreach (Node n in r.GetNodes())
b.Write(GetBytes(n));
}
return m.ToArray();
}
}
public static byte[] GetBytes(Tag tag)
{
using (MemoryStream m = new MemoryStream())
{
using (BinaryWriter b = new BinaryWriter(m))
{
byte[] valueBytes;
if (tag.value is ulong valueUlong)
{
valueBytes = BitConverter.GetBytes(valueUlong);
}else if (tag.value is byte valueByte)
{
valueBytes = new[] { valueByte };
}else if (tag.value is bool valueBool)
{
valueBytes = BitConverter.GetBytes(valueBool);
}else if (tag.value is string valueString)
{
valueBytes = Encoding.ASCII.GetBytes(valueString);
}else if (tag.value is int valueInt)
{
valueBytes = BitConverter.GetBytes(valueInt);
}
else
{
throw new ArgumentException($"Unknown type {tag.value.GetType()}");
}
b.Write(sizeof(byte) + valueBytes.Length);
b.Write((byte)tag.key);
b.Write(valueBytes);
}
return m.ToArray();
}
}
public static byte[] GetBytes(TagManager t)
{
using (MemoryStream m = new MemoryStream())
{
using (BinaryWriter b = new BinaryWriter(m))
{
HashSet<byte[]> wayBytesSet = new HashSet<byte[]>();
int totalBytes = 0;
foreach (KeyValuePair<ulong, HashSet<Tag>> way in t.wayTags)
{
HashSet<byte[]> tagBytesSet = new HashSet<byte[]>();
tagBytesSet.Add(BitConverter.GetBytes(way.Key));
int totalWayBytes = sizeof(ulong);
foreach (Tag tag in way.Value)
{
byte[] tagBytes = GetBytes(tag);
totalWayBytes += tagBytes.Length;
tagBytesSet.Add(tagBytes);
}
wayBytesSet.Add(BitConverter.GetBytes(totalWayBytes));
foreach (byte[] bytes in tagBytesSet)
{
wayBytesSet.Add(bytes);
}
totalBytes += sizeof(int) + totalWayBytes;
}
b.Write(totalBytes);
foreach (byte[] bytes in wayBytesSet)
{
b.Write(bytes);
}
}
return m.ToArray();
}
}
}

71
OsmXmlToRegionConverter/Coordinates.cs
View File

@ -1,71 +0,0 @@
namespace OsmXmlToRegionConverter;
public class Coordinates
{
public const float RegionSize = 0.01f;
public float latitude { get; }
public float longitude { get; }
public Coordinates(float latitude, float longitude)
{
this.latitude = latitude;
this.longitude = longitude;
}
public override bool Equals(object? obj)
{
if (obj == null || obj.GetType() != this.GetType())
return false;
Coordinates convObj = (Coordinates)obj;
// ReSharper disable twice CompareOfFloatsByEqualityOperator static values
return convObj.latitude == this.latitude && convObj.longitude == this.longitude;
}
public new ulong GetHashCode()
{
return GetHashCode(latitude, longitude);
}
public static ulong GetRegionHashCode(float latitude, float longitude)
{
float latRegion = latitude - latitude % RegionSize;
float lonRegion = longitude - longitude % RegionSize;
return GetHashCode(latRegion, lonRegion);
}
public static ulong GetRegionHashCode(Coordinates coordinates)
{
float latRegion = coordinates.latitude - coordinates.latitude % RegionSize;
float lonRegion = coordinates.longitude - coordinates.longitude % RegionSize;
return GetHashCode(latRegion, lonRegion);
}
private const float decimalCoordsSave = 10000;
private const ulong offset = 10000000;
//Latitude maxChars = 7
//Longitude maxChars = 8
public static ulong GetHashCode(float latitude, float longitude)
{
ulong latHash = Convert.ToUInt64((latitude + 90) * decimalCoordsSave);
ulong lonHash = Convert.ToUInt64((longitude + 180) * decimalCoordsSave);
return latHash * offset + lonHash;
}
public override string ToString()
{
return string.Format("COORDINATES Lat: {0} Lon: {1}", this.latitude, this.longitude);
}
public const int ByteSize = sizeof(float) * 2;
public static Coordinates FromBytes(byte[] bytes)
{
using (MemoryStream m = new MemoryStream(bytes)) {
using (BinaryReader r = new BinaryReader(m))
{
float latitude = r.ReadSingle();
float longitude = r.ReadSingle();
return new Coordinates(latitude, longitude);
}
}
}
}

36
OsmXmlToRegionConverter/Edge.cs
View File

@ -1,36 +0,0 @@
namespace OsmXmlToRegionConverter;
public class Edge
{
public ulong wayId { get; }
public ulong neighborId { get; }
public ulong neighborRegion { get; }
public Edge(ulong wayId, ulong neighborId, ulong neighborRegion)
{
this.wayId = wayId;
this.neighborId = neighborId;
this.neighborRegion = neighborRegion;
}
public Edge(ulong wayId, Node neighborNode)
{
this.wayId = wayId;
this.neighborId = neighborNode.id;
this.neighborRegion = Coordinates.GetRegionHashCode(neighborNode.coordinates);
}
public const int ByteSize = sizeof(ulong) * 3;
public static Edge FromBytes(byte[] bytes)
{
using (MemoryStream m = new MemoryStream(bytes)) {
using (BinaryReader r = new BinaryReader(m))
{
ulong wayId = r.ReadUInt64();
ulong neighborId = r.ReadUInt64();
ulong neighborRegion = r.ReadUInt64();
return new Edge(wayId, neighborId, neighborRegion);
}
}
}
}

228
OsmXmlToRegionConverter/Manager.cs
View File

@ -1,228 +0,0 @@
using System.Globalization;
using System.Xml;
namespace OsmXmlToRegionConverter;
public class Manager
{
private static readonly XmlReaderSettings readerSettings = new()
{
IgnoreWhitespace = true,
IgnoreComments = true
};
private static readonly NumberFormatInfo decimalInfo = new()
{
NumberDecimalSeparator = "."
};
private const string NodesTmpFilename = "nodes.tmp";
private const string WaysTmpFilename = "ways.tmp";
private const string TagsFilename = "tags";
public static void XmlToByteConverter(string inputFilePath, string outputFolderPath)
{
if (!File.Exists(inputFilePath))
throw new FileNotFoundException(inputFilePath);
Directory.CreateDirectory(outputFolderPath);
Console.WriteLine("Getting highwayNodeIDs...");
FileStream xmlFileStream = new FileStream(inputFilePath, FileMode.Open);
HashSet<ulong> nodeIdsToImport = GetHighwayNodeIds(XmlReader.Create(xmlFileStream, readerSettings));
xmlFileStream.Position = 0;
Console.WriteLine("Converting Nodes...");
Dictionary<ulong, ulong> nodeRegions = ImportNodesToRegionsTmp(XmlReader.Create(xmlFileStream, readerSettings), nodeIdsToImport, outputFolderPath);
nodeIdsToImport.Clear(); //memory management
xmlFileStream.Position = 0;
Console.WriteLine("Converting Ways...");
ImportWaysToRegionTmp(XmlReader.Create(xmlFileStream, readerSettings), nodeRegions, outputFolderPath);
HashSet<ulong> regionHashes = nodeRegions.Values.ToHashSet();
nodeRegions.Clear(); //memory management
Console.WriteLine("Converting Nodes and Ways to Nodes and Edges in regionfile...");
ConvertNodesAndWaysToRegion(outputFolderPath, regionHashes);
}
private static HashSet<ulong> GetHighwayNodeIds(XmlReader xmlReader)
{
HashSet<ulong> retSet = new();
bool isHighway;
HashSet<ulong> currentIds = new();
while (xmlReader.ReadToFollowing("way"))
{
isHighway = false;
currentIds.Clear();
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway"))
{
isHighway = true;
}
else if (xmlReader.Name == "nd")
{
try
{
currentIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
}
catch (FormatException) { };
}
}
if (isHighway)
{
retSet.UnionWith(currentIds);
}
wayReader.Close();
}
xmlReader.Close();
return retSet;
}
private static Dictionary<ulong, ulong> ImportNodesToRegionsTmp(XmlReader xmlReader, HashSet<ulong> nodeIdsToImport, string outputFolderPath)
{
Dictionary<ulong, FileStream> regionWriters = new Dictionary<ulong, FileStream>();
Dictionary<ulong, ulong> nodeRegions = new Dictionary<ulong, ulong>();
while (xmlReader.ReadToFollowing("node"))
{
ulong currentNodeId = Convert.ToUInt64(xmlReader.GetAttribute("id"));
if (nodeIdsToImport.Contains(currentNodeId))
{
float lat = Convert.ToSingle(xmlReader.GetAttribute("lat")!, decimalInfo);
float lon = Convert.ToSingle(xmlReader.GetAttribute("lon")!, decimalInfo);
TmpNode newTmpNode = new TmpNode(currentNodeId, lat, lon);
ulong regionHash = Coordinates.GetRegionHashCode(newTmpNode.coordinates);
if (!regionWriters.ContainsKey(regionHash))
{
Directory.CreateDirectory(Path.Combine(outputFolderPath, regionHash.ToString()));
regionWriters.Add(regionHash, new FileStream(Path.Combine(outputFolderPath, regionHash.ToString(), NodesTmpFilename), FileMode.Append));
}
regionWriters[regionHash].Write( ByteConverter.GetBytes(newTmpNode));
nodeRegions.Add(currentNodeId, regionHash);
}
}
foreach(FileStream regionWriter in regionWriters.Values)
regionWriter.Close();
xmlReader.Close();
return nodeRegions;
}
private static void ImportWaysToRegionTmp(XmlReader xmlReader, Dictionary<ulong, ulong> nodeRegions, string outputFolderPath)
{
Dictionary<ulong, FileStream> regionWriters = new Dictionary<ulong, FileStream>();
bool isHighway = false;
HashSet<ulong> currentIds = new();
while (xmlReader.ReadToFollowing("way"))
{
HashSet<Tag> tagsSet = new HashSet<Tag>();
isHighway = false;
currentIds.Clear();
ulong wayId = Convert.ToUInt64(xmlReader.GetAttribute("id"));
XmlReader wayReader = xmlReader.ReadSubtree();
while (wayReader.Read())
{
if (xmlReader.Name == "tag" && xmlReader.GetAttribute("k")!.Equals("highway"))
{
isHighway = true;
}
else if (xmlReader.Name == "nd")
{
try
{
currentIds.Add(Convert.ToUInt64(xmlReader.GetAttribute("ref")));
}
catch (FormatException) { };
}else if (xmlReader.Name == "tag")
{
tagsSet.Add(Tag.ConvertToTag(xmlReader.GetAttribute("k")!, xmlReader.GetAttribute("v")!));
}
}
if (isHighway)
{
TmpWay newTmpWay = new TmpWay(wayId, currentIds.ToArray(), tagsSet);
HashSet<ulong> regionsToWrite = new HashSet<ulong>();
foreach (ulong nodeId in currentIds)
{
if (nodeRegions.ContainsKey(nodeId))
{
regionsToWrite.Add(nodeRegions[nodeId]);
}
}
foreach (ulong regionHash in regionsToWrite)
{
if (!regionWriters.ContainsKey(regionHash))
{
regionWriters.Add(regionHash, new FileStream(Path.Combine(outputFolderPath, regionHash.ToString(), WaysTmpFilename), FileMode.Append));
}
regionWriters[regionHash].Write(ByteConverter.GetBytes(newTmpWay));
}
}
wayReader.Close();
}
foreach(FileStream regionWriter in regionWriters.Values)
regionWriter.Close();
xmlReader.Close();
}
private static void ConvertNodesAndWaysToRegion(string outputFolderPath, HashSet<ulong> regionHashes)
{
foreach (ulong regionHash in regionHashes)
{
Console.WriteLine($"Converting Region {regionHash}");
string regionFilePath = Path.Combine(outputFolderPath, $"{regionHash.ToString()}.region");
string regionFolderPath = Path.Combine(outputFolderPath, regionHash.ToString());
string nodesFilePath = Path.Combine(regionFolderPath, NodesTmpFilename);
string waysFilePath = Path.Combine(regionFolderPath, WaysTmpFilename);
Region newRegion = new Region(regionHash);
Dictionary<ulong, Node> nodes = new Dictionary<ulong, Node>();
byte[] nodeBytes = File.ReadAllBytes(nodesFilePath);
HashSet<object> nodeObjects = ByteConverter.GetObjectsFromBytes(nodeBytes);
foreach (object obj in nodeObjects)
{
if (obj.GetType() == typeof(TmpNode))
{
TmpNode tmpNode = (TmpNode)obj;
Node newNode = new Node(tmpNode.id, tmpNode.coordinates);
newRegion.AddNode(newNode);
nodes.TryAdd(newNode.id, newNode);
}
else
throw new Exception("Unexpected object.");
}
byte[] wayBytes = File.ReadAllBytes(waysFilePath);
HashSet<object> ways = ByteConverter.GetObjectsFromBytes(wayBytes);
foreach (object obj in ways)
{
if (obj.GetType() == typeof(TmpWay))
{
TmpWay way = (TmpWay)obj;
for (int nodeIdi = 0; nodeIdi < way.wayNodeIds.Length - 1; nodeIdi++) //TODO directional
{
if (nodes.ContainsKey(way.wayNodeIds[nodeIdi]) &&
nodes.ContainsKey(way.wayNodeIds[nodeIdi + 1]))
{
Node n1 = nodes[way.wayNodeIds[nodeIdi]];
Node n2 = nodes[way.wayNodeIds[nodeIdi + 1]];
n1.edges.Add(new Edge(way.id, n2));
n2.edges.Add(new Edge(way.id, n1));
}
}
}
else throw new Exception("Unexpected object.");
}
File.WriteAllBytes(regionFilePath, ByteConverter.GetBytes(newRegion));
Directory.Delete(regionFolderPath, true);
}
}
}

48
OsmXmlToRegionConverter/Node.cs
View File

@ -1,48 +0,0 @@
namespace OsmXmlToRegionConverter;
public class Node
{
public ulong id { get; }
public Coordinates coordinates { get; }
public HashSet<Edge> edges { get; }
public Node(ulong id, float latitude, float longitude)
{
this.id = id;
coordinates = new Coordinates(latitude, longitude);
edges = new HashSet<Edge>();
}
public Node(ulong id, Coordinates coordinates)
{
this.id = id;
this.coordinates = coordinates;
edges = new HashSet<Edge>();
}
public void AddEdge(Edge edge)
{
edges.Add(edge);
}
public const int ByteSizeEmpty = sizeof(int) + sizeof(ulong) + Coordinates.ByteSize;
public static Node FromBytes(byte[] bytes)
{
using (MemoryStream m = new MemoryStream(bytes)) {
using (BinaryReader r = new BinaryReader(m))
{
int edgeCount = r.ReadInt32();
ulong id = r.ReadUInt64();
byte[] coordinateBytes = r.ReadBytes(sizeof(float) * 2);
Coordinates coordinates = Coordinates.FromBytes(coordinateBytes);
Node ret = new Node(id, coordinates);
for (int i = 0; i < edgeCount; i++)
ret.AddEdge(Edge.FromBytes(r.ReadBytes(Edge.ByteSize)));
return ret;
}
}
}
}

9
OsmXmlToRegionConverter/OsmXmlToRegionConverter.csproj
View File

@ -1,9 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

47
OsmXmlToRegionConverter/Region.cs
View File

@ -1,47 +0,0 @@
namespace OsmXmlToRegionConverter;
public class Region
{
private HashSet<Node> nodes;
public ulong regionHash { get; }
private TagManager tagManager { get; }
public Region(ulong regionHash)
{
this.regionHash = regionHash;
nodes = new HashSet<Node>();
tagManager = new TagManager();
}
public void AddNode(Node nodeToAdd)
{
if (nodes.Contains(nodeToAdd))
throw new Exception("Node already in region");
else
nodes.Add(nodeToAdd);
}
public Node[] GetNodes()
{
return this.nodes.ToArray();
}
public const int ByteSizeEmpty = sizeof(int) + sizeof(ulong);
public static Region FromBytes(byte[] bytes)
{
using (MemoryStream m = new MemoryStream(bytes)) {
using (BinaryReader r = new BinaryReader(m))
{
int nodesInRegionCount = r.ReadInt32();
ulong regionHash = r.ReadUInt64();
Region retRegion = new Region(regionHash);
HashSet<object> nodesSet =
ByteConverter.GetObjectsFromBytes(bytes.SubArray(ByteSizeEmpty, bytes.Length - ByteSizeEmpty));
foreach(object node in nodesSet)
retRegion.AddNode((Node)node);
return retRegion;
}
}
}
}

70
OsmXmlToRegionConverter/TagManager.cs
View File

@ -1,70 +0,0 @@
using System.Runtime.CompilerServices;
namespace OsmXmlToRegionConverter;
public class TagManager
{
public readonly Dictionary<ulong, HashSet<Tag>> wayTags = new();
public bool ContainsKey(ulong wayId, Tag.TagType key)
{
return wayTags.ContainsKey(wayId) && wayTags[wayId].Any(tag => tag.key == key);
}
public object? GetTag(ulong wayId, Tag.TagType key)
{
return ContainsKey(wayId, key) ? wayTags[wayId].First(tag => tag.key == key) : null;
}
public void AddTag(ulong wayId, string key, string value)
{
Tag tag = Tag.ConvertToTag(key, value);
AddTag(wayId, tag);
}
private void AddTag(ulong wayId, Tag tag)
{
if (tag.key != Tag.TagType.EMPTY)
{
if(!wayTags.ContainsKey(wayId))
wayTags.Add(wayId, new HashSet<Tag>());
wayTags[wayId].Add(tag);
}
}
public static TagManager FromBytes(byte[] bytes)
{
TagManager newTagManager = new TagManager();
using (MemoryStream m = new MemoryStream(bytes)) {
using (BinaryReader r = new BinaryReader(m))
{
while (r.BaseStream.Position < bytes.Length)
{
int wayByteSize = r.ReadInt32();
byte[] wayBytes = r.ReadBytes(wayByteSize);
AddWayFromBytes(ref newTagManager, wayBytes);
}
}
}
return newTagManager;
}
private static void AddWayFromBytes(ref TagManager tagManager, byte[] bytes)
{
using (MemoryStream m = new MemoryStream(bytes)) {
using (BinaryReader r = new BinaryReader(m))
{
ulong wayId = r.ReadUInt64();
while (r.BaseStream.Position < bytes.Length)
{
int tagsByteSize = r.ReadInt32();
byte[] tagBytes = r.ReadBytes(tagsByteSize);
Tag tag = Tag.FromBytes(tagBytes);
tagManager.AddTag(wayId, tag);
}
}
}
}
}

35
OsmXmlToRegionConverter/TmpNode.cs
View File

@ -1,35 +0,0 @@
namespace OsmXmlToRegionConverter;
public class TmpNode
{
public ulong id { get; }
public Coordinates coordinates { get; }
public TmpNode(ulong id, float latitude, float longitude)
{
this.id = id;
this.coordinates = new Coordinates(latitude, longitude);
}
public TmpNode(ulong id, Coordinates coordinates)
{
this.id = id;
this.coordinates = coordinates;
}
public const int ByteSize = sizeof(ulong) + Coordinates.ByteSize;
public static TmpNode FromBytes(byte[] bytes)
{
using (MemoryStream m = new MemoryStream(bytes)) {
using (BinaryReader r = new BinaryReader(m))
{
ulong id = r.ReadUInt64();
byte coordinatesType = r.ReadByte();
byte[] coordinateBytes = r.ReadBytes(sizeof(float) * 2);
Coordinates coordinates = Coordinates.FromBytes(coordinateBytes);
return new TmpNode(id, coordinates);
}
}
}
}

31
OsmXmlToRegionConverter/TmpWay.cs
View File

@ -1,31 +0,0 @@
namespace OsmXmlToRegionConverter;
public class TmpWay
{
public ulong id { get; }
public ulong[] wayNodeIds { get; }
public HashSet<Tag> tags { get; }
public TmpWay(ulong id, ulong[] wayNodeIds, HashSet<Tag> tags)
{
this.id = id;
this.wayNodeIds = wayNodeIds;
this.tags = tags;
}
public static TmpWay FromBytes(byte[] bytes)
{
throw new NotImplementedException();/*
using (MemoryStream m = new MemoryStream(bytes)) {
using (BinaryReader r = new BinaryReader(m))
{
int wayNodeIdsLength = r.ReadInt32();
ulong id = r.ReadUInt64();
ulong[] wayNodeIds = new ulong[wayNodeIdsLength];
for (int i = 0; i < wayNodeIds.Length; i++)
wayNodeIds[i] = r.ReadUInt64();
return new TmpWay(id, wayNodeIds);
}
}*/
}
}