importer.cpp

Go to the documentation of this file.

#include <boost/filesystem/path.hpp>
#include <boost/unordered_map.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/type_traits/is_same.hpp>
#include <unordered_set>

#include "../../extras/eaglexml/eaglexml.hpp"
#include "../../extras/eaglexml/eaglexml_iterators.hpp"

#include "importer/importer.hpp"

#include "general/configuration.hpp"
#include "general/geodata.hpp"

#include "general/geo_object.hpp"
#include "general/node.hpp"
#include "general/way.hpp"
#include "general/relation.hpp"


using boost::filesystem::path;
using nl = std::numeric_limits<double>;


class Importer::OsmXmlParser
    : public eaglexml::stream_cache<>::stream_cache_observer
{
public:
    static const int parser_flags = eaglexml::parse_no_data_nodes
                                    | eaglexml::parse_normalize_whitespace
                                    | eaglexml::parse_trim_whitespace
                                    | eaglexml::parse_validate_closing_tags
                                    | eaglexml::parse_default;

    typedef uint64_t OsmIdType;

public:
    OsmXmlParser(bool ignoreUnknownEntities, const FloatRect& bounds = { -nl::max(), -nl::max(), nl::max(), nl::max() })
        : ignoreUnknownEntities(ignoreUnknownEntities)
        , clippingBounds(bounds)
        , alreadyRead(0)
        , fileSize(0)
        , segmentSize(1024 * 1024)
        , outputIgnoreRelation(false)
        , outputIgnoreBounds(false)
    {
    }

    void parse(const path& xml_file)
    {
        assert(!nodes);
        assert(!ways);
        assert(!relations);

        std::ifstream xml_stream(xml_file.string());

        if(!xml_stream)
            BOOST_THROW_EXCEPTION(excp::FileNotFoundException()  << excp::InfoFileName(xml_file.string()));

        LOG_SEV(importer_log, info) << "Load xml-file \"" << xml_file.string() << "\"";
        
        fileSize = boost::filesystem::file_size(xml_file);

        LOG_SEV(importer_log, info) << "File size is " << fileSize / (1024) << "kb";

        nodes = boost::make_shared<std::vector<Node> >();
        ways = boost::make_shared<std::vector<Way> >();
        relations = boost::make_shared<std::vector<Relation> >();

        // Use a cache with an 8 mb buffer
        eaglexml::stream_cache<> cache(xml_stream, 8 * 1024 * 1024);
        eaglexml::xml_document<> document;

        cache.segment_size(segmentSize);
        cache.max_segments_to_read(1);
        cache.observer(this);


        try {
            // Init the xml parser
            document.parse<parser_flags>(&cache);

            eaglexml::xml_node<>* osm = document.first_node("osm");

            if(!osm)
                BOOST_THROW_EXCEPTION(excp::InputFormatException() << excp::InfoXmlEntityName("<osm>")  << excp::InfoWhat("Missing \"<osm>\" xml node!"));

            parseEntities(osm);

        }catch(excp::InputFormatException& e) {
            e << excp::InfoFileName(xml_file.string());
            throw;
        }catch(eaglexml::parse_error& e)
        {
            BOOST_THROW_EXCEPTION(excp::InputFormatException() << excp::InfoFileName(xml_file.string()) << excp::InfoWhat(e.what()));
        }
    }

    shared_ptr< std::vector<Node> > getParsedNodes() const
    {
        assert(nodes);
        return nodes;
    }
    
    shared_ptr< std::vector<Way> > getParsedWays() const
    {
        assert(ways);
        return ways;
    }
    
    shared_ptr< std::vector<Relation> > getParsedRelations() const
    {
        assert(relations);
        return relations;
    }

    std::size_t getNumberOfClippedNodes() const
    {
        return clippedNodes.size();
    }
    
private:
    void parseEntities(eaglexml::xml_node<>* osmRoot)
    {
        assert(osmRoot);

        std::map<std::string, void (OsmXmlParser::*)(eaglexml::xml_node<>*)> entities;

        // Entities that can be parsed
        entities["bounds"] = &OsmXmlParser::parseBounds;
        entities["node"] = &OsmXmlParser::parseNode;
        entities["way"] = &OsmXmlParser::parseWay;
        entities["relation"] = &OsmXmlParser::parseRelation;

        for(eaglexml::node_iterator<> it(osmRoot);
            it != eaglexml::node_iterator<>();
            ++it)
        {
            auto entityIt = entities.find(it->name());

            if(!ignoreUnknownEntities && entityIt == entities.end())
                BOOST_THROW_EXCEPTION(excp::InputFormatException() << excp::InfoXmlEntityName(it->name()) << excp::InfoWhat("Unknown entity in xml file!"));

            try{ 
                (this->*(entityIt->second))(&*it);
            }catch(excp::BadOsmIdException& e) {
                const auto id = *boost::get_error_info<excp::InfoUnresolvableId>(e);
                if (!clippedNodes.count(id)) {
                    LOG_SEV(importer_log, warning) << "Bad osm id[" << id << "]. Entity is skipped!";
                }
            }
        }
    }

    void parseBounds(eaglexml::xml_node<>* node)
    {
        if(!outputIgnoreBounds)
        {
            LOG_SEV(importer_log, info) << "Bounds tag in osm data is ignored by this software!";
            outputIgnoreBounds = true;
        }
    }

    void parseNode(eaglexml::xml_node<>* node)
    {
        assert(node);

        OsmIdType id;
        double lon, lat;
        extractAttributeFromNode("id", node, &id);
        extractAttributeFromNode("lon", node, &lon);
        extractAttributeFromNode("lat", node, &lat);

        FloatPoint loc = {lon, lat};
        if (clippingBounds.contains(loc)) {
            DataMap<CachedString, CachedString> tags;
            parseProperties<Node>(node->first_node(), &tags, nullptr, nullptr, nullptr, nullptr);

            nodeIdMapping.insert(std::make_pair(id, NodeId(nodes->size())));
            nodes->push_back(Node(loc, tags));
        } else {
            clippedNodes.insert(id);
        }
    }
    
    void parseWay(eaglexml::xml_node<>* way)
    {
        assert(way);

        OsmIdType id;
        extractAttributeFromNode("id", way, &id);


        DataMap<CachedString, CachedString> tags;
        std::vector<NodeId> nodeIds;

        parseProperties<Way>(way->first_node(), &tags, &nodeIds, nullptr, nullptr, nullptr);

        if (nodeIds.size() == 0)
            return;

        wayIdMapping.insert(std::make_pair(id, WayId(ways->size())));
        ways->push_back(Way(nodeIds, tags));
    }
    
    void parseRelation(eaglexml::xml_node<>* relation)
    {
        assert(relation);

        OsmIdType id;
        extractAttributeFromNode("id", relation, &id);

        DataMap<CachedString, CachedString> tags;
        std::vector<NodeId> nodeIds;
        std::vector<WayId> wayIds;
        DataMap<NodeId, CachedString> nodeRoles;
        DataMap<WayId, CachedString> wayRoles;


        parseProperties<Relation>(relation->first_node(), &tags, &nodeIds, &nodeRoles, &wayIds, &wayRoles);

        if (nodeIds.size() == 0 && wayIds.size() == 0)
            return;

        relations->push_back(Relation(nodeIds, nodeRoles, wayIds, wayRoles, tags));
    }

    
    template<typename Target>
    inline void parseProperties(eaglexml::xml_node<>* firstProp,
                                DataMap<CachedString, CachedString>* tagMap,
                                std::vector<NodeId>* nodeRefIds,
                                DataMap<NodeId, CachedString>* nodeRoleMap,
                                std::vector<WayId>* wayRefIds,
                                DataMap<WayId, CachedString>* wayRoleMap)
    {
        BOOST_STATIC_ASSERT((boost::is_same<Target, Node>::value || boost::is_same<Target, Way>::value || boost::is_same<Target, Relation>::value));
        assert(tagMap);

        // Go through all properties and parse them
        eaglexml::xml_node<>* prop = firstProp;
        for(;prop; prop = prop->next_sibling())
        {
            const char* propName = firstProp->name();

            if(std::strcmp(propName, "tag") == 0)
            {
                // Parse tag
                CachedString key;
                CachedString value;
                extractAttributeFromNode("k", prop, &key);
                extractAttributeFromNode("v", prop, &value);

                (*tagMap)[key] = value;

            }else if(boost::is_same<Target, Way>::value && std::strcmp(propName, "nd") == 0)
            {
                assert(nodeRefIds);

                // Parse a node reference
                OsmIdType osmId;
                extractAttributeFromNode("ref", prop, &osmId);
                nodeRefIds->push_back(resolveOsmId(osmId, nodeIdMapping));
            }else if(boost::is_same<Target, Relation>::value && std::strcmp(propName, "member") == 0)
            {
                // Parse a relation member
                assert(nodeRefIds);
                assert(nodeRoleMap);
                assert(wayRefIds);
                assert(wayRoleMap);


                eaglexml::xml_attribute<>* attr = prop->first_attribute("type");

                if(!attr || attr->value_size() == 0)
                    BOOST_THROW_EXCEPTION(excp::InputFormatException() << excp::InfoXmlEntityName("relation::member::type"));

                OsmIdType osmRefId;
                extractAttributeFromNode("ref", prop, &osmRefId);

                // Evaluate type attribute
                const char* type = attr->value();
                if(std::strcmp(type, "node") == 0)
                {
                    NodeId nodeId = resolveOsmId(osmRefId, nodeIdMapping);
                    nodeRefIds->push_back(nodeId);
                    extractAttributeFromNode("role", prop, &((*nodeRoleMap)[nodeId]));
                }else if(std::strcmp(type, "way") == 0)
                {
                    WayId wayId = resolveOsmId(osmRefId, wayIdMapping);
                    wayRefIds->push_back(wayId);
                    extractAttributeFromNode("role", prop, &((*wayRoleMap)[wayId]));
                }else if(std::strcmp(type, "relation") == 0) {
                    
                    if(!outputIgnoreRelation)
                    {
                        LOG_SEV(importer_log, warning) << "This software does not support relation member in relations!";
                        LOG_SEV(importer_log, warning) << "Reference is ignored!";
                        outputIgnoreRelation = true;
                    }
                    continue;
                }else{

                    BOOST_THROW_EXCEPTION(excp::InputFormatException() << excp::InfoXmlEntityName("relation::member::type::value"));
                }

            }
        }
    }

    template<typename T>
    void extractAttributeFromNode(const string& attrname, eaglexml::xml_node<>* node, T* dest)
    {
        assert(node);
        assert(dest);

        eaglexml::xml_attribute<>* attr = node->first_attribute(attrname.c_str(), attrname.size());

        if(!attr)
            BOOST_THROW_EXCEPTION(excp::InputFormatException() << excp::InfoXmlEntityName(attrname));

        const char* value = attr->value();
        try {
            *dest = boost::lexical_cast<T>(value);
        }catch(boost::bad_lexical_cast& e)
        {
            (void)e;
            BOOST_THROW_EXCEPTION(excp::InputFormatException() << excp::InfoXmlEntityName(attrname) << excp::InfoBadSourceValue(value));
        }
    }

    template<typename IdType>
    IdType resolveOsmId(OsmIdType osmId, const boost::unordered_map<OsmIdType, IdType>& table)
    {
        auto it = table.find(osmId);

        if(it == table.end())
            BOOST_THROW_EXCEPTION(excp::BadOsmIdException() << excp::InfoUnresolvableId(osmId));

        return it->second;
    }


    virtual void on_fetch( unsigned int chars_left, unsigned int need, node_type* active_node ) {}
    virtual void on_buffer_resize() {}
    virtual void on_segment_read() {}

    virtual void on_read_begin( unsigned int segments )
    {
        int before = int(100 * (double)alreadyRead / (double)fileSize);
        alreadyRead += segments * segmentSize;
        int after  = int(100 * (double)alreadyRead / (double)fileSize);

        if(after != before)
            LOG_SEV(importer_log, info) << "Loading [" << std::min(after, 99) << "%]";
    }



private:

    bool ignoreUnknownEntities;

    const FloatRect clippingBounds;
    
    boost::unordered_map<OsmIdType, NodeId> nodeIdMapping;

    boost::unordered_map<OsmIdType, WayId>  wayIdMapping;

    std::unordered_set<OsmIdType> clippedNodes;

    shared_ptr< std::vector<Node> > nodes;

    shared_ptr< std::vector<Way> > ways;

    shared_ptr< std::vector<Relation> > relations;

    std::uintmax_t  fileSize;

    std::uintmax_t  alreadyRead;

    unsigned int    segmentSize;

    bool outputIgnoreRelation, outputIgnoreBounds;
};









Importer::Importer(const shared_ptr<Configuration>& config)
    : config(config)
{
}

shared_ptr<Geodata> Importer::importXML()
{
    FloatRect bounds = {
        FloatPoint(config->get(opt::importer::min_lon, -nl::max()), config->get(opt::importer::min_lat, -nl::max())),
        FloatPoint(config->get(opt::importer::max_lon, nl::max()), config->get(opt::importer::max_lat, nl::max()))
    };
    LOG_SEV(importer_log, info) << "Clipping nodes with [lon: " << bounds.minX << " to " << bounds.maxX << ", lat: " << bounds.minY << " to " << bounds.maxY << "]";
    OsmXmlParser parser(!config->get<bool>(opt::importer::check_xml_entities), bounds);
    shared_ptr<Geodata> geodata = boost::make_shared<Geodata>();

    path xml_file = config->get<string>(opt::importer::path_to_osmdata);
    LOG_SEV(importer_log, info) << "Start parsing...";
    parser.parse(xml_file);

    LOG_SEV(importer_log, info) << "Insert into geodata...";
    geodata->insertNodes(parser.getParsedNodes());
    geodata->insertWays(parser.getParsedWays());
    geodata->insertRelations(parser.getParsedRelations());

    const auto node_count = parser.getParsedNodes()->size();
    LOG_SEV(importer_log, info) << "Clipped " << parser.getNumberOfClippedNodes() << " / " << (parser.getNumberOfClippedNodes() + node_count) << " nodes. " << node_count << " nodes remaining.";

    return geodata;
}