Separate class for properties to handle protected and web-fields.

convert.py

@@ -39,7 +39,6 @@ uuid_map = {}
 
 for item in opif:
 
-    props = item.get_all_props()
 
     # Fields that are not to be added as custom properties
     fids_done = ["passwordHistory"]
@@ -52,11 +51,12 @@ for item in opif:
         target_group_name = "Recycle Bin"
 
     # Add entry to KeePass
-    entry = kp.add_entry(target_group_name, props["title"])
+    entry = kp.add_entry(target_group_name, item.get_property("title").value)
     fids_done.append("title")
 
-    # UUID - memorise for later linking?
+    # UUID - memorise for later linking (if supported by output format)?
-    uuid_map[props["uuid"]] = entry.uuid
+    uuid = item.get_property("uuid").value
+    uuid_map[uuid] = entry.uuid
     fids_done.append("uuid")
 
     # Icon
@@ -64,16 +64,19 @@ for item in opif:
     kp.set_icon(kp_icon)
 
     # URLs
-    if "location" in props:
+    location = item.get_property("location")
-        kp.add_url(props["location"])
+    if location:
+        kp.add_url(location.value)
         fids_done.append("location")
         fids_done.append("locationKey")
-    if "URLs" in props:
+    urls = item.get_property("URLs")
-        for u in props["URLs"]:
+    if urls:
+        for u in urls.raw:
             kp.add_url(u["url"])
         fids_done.append("URLs")
-    if "URL" in props:
+    url = item.get_property("URL")
-        kp.add_url(props["URL"])
+    if url:
+        kp.add_url(url.value)
         fids_done.append("URL")
 
     # Tags
@@ -87,13 +90,15 @@ for item in opif:
             kp.add_totp(totp[0], title=totp[1])
 
     # Notes
-    if "notesPlain" in props:
+    notes_plain = item.get_property("notesPlain")
-        entry.notes = props["notesPlain"]
+    print("Notes: {}".format(repr(notes_plain)))
+    if notes_plain:
+        entry.notes = notes_plain.raw
         fids_done.append("notesPlain")
 
     # Dates
-    entry.ctime = datetime.datetime.fromtimestamp(props["createdAt"])
+    entry.ctime = datetime.datetime.fromtimestamp(item.get_property("createdAt").raw)
-    entry.mtime = datetime.datetime.fromtimestamp(props["updatedAt"])
+    entry.mtime = datetime.datetime.fromtimestamp(item.get_property("updatedAt").raw)
     fids_done.append("createdAt")
     fids_done.append("updatedAt")
 
@@ -105,41 +110,42 @@ for item in opif:
         if type(seek_fields) is str:
             seek_fields = [seek_fields]
         for fid in seek_fields:
-            if fid in props:
+            prop = item.get_property(fid)
-                setattr(entry, map_field, props[fid])
+            if prop:
+                setattr(entry, map_field, prop.value)
                 fids_done.append(fid)
                 break
 
     # Set remaining properties
-    for k, v in props.items():
+    for k in item.get_property_keys():
         if k in ["Password"]:
             # Forbidden name
             continue
+        if k[:5] == "TOTP_":
+            # Skip OTPs as they're handled separately
+            continue
         if k in RECORD_MAP["General"]["ignored"]:
             # Skip ignored fields
             continue
         if k in fids_done:
             # Skip fields processed elsewhere
             continue
-        kp.set_prop(k, str(v))
+        v = item.get_property(k)
+        if v.is_web_field:
+            kp.set_prop("KPH: {}".format(v.title), v.value, protected=v.is_protected)
+        else:
+            kp.set_prop(v.title, v.value, protected=v.is_protected)
 
 
 
    # TODO: scope: Never = never suggest in browser (i.e. don't add KPH fields)
 
-    secure = item.raw["secureContents"]
-    # Other web fields
-    if "fields" in secure:
-        for field in secure["fields"]:
-            d = field.get("designation")
-            if d != "username" and d != "password":
-                entry.set_custom_property("KPH: {}".format(field["name"]), field["value"])
-
     # AFTER ALL OTHER PROCESSING IS DONE: Password history
-    if "passwordHistory" in props:
+    password_history = item.get_property("passwordHistory")
+    if password_history:
         original_password = entry.password
         original_mtime = entry.mtime
-        for p in props["passwordHistory"]:
+        for p in password_history.raw:
             d = datetime.datetime.fromtimestamp(p["time"])
             entry.mtime = d
             entry.password = p["value"]

kpwriter.py

@@ -51,7 +51,7 @@ class KpWriter:
             suffix = "_{}".format(suffix_ctr)
 
         self.set_prop("TimeOtp-Secret-Base32{}".format(suffix), init_string, True)
-        self.set_prop("otp{}".format(suffix), otp_url)
+        self.set_prop("otp{}".format(suffix), otp_url, True)
         if len(title) > 0:
             self.set_prop("otp_title{}".format(suffix), title)
 

mappings.yml

@@ -5,6 +5,7 @@ General:
         - securityLevel
         - typeName
         - uuid
+        - trashed
 
 # Mapping of 1P types for best conversion
 

onepif/OnepifEntry.py

@@ -1,5 +1,5 @@
 import sys
-from datetime import datetime
+from . import OnepifEntryProperty as oep
 
 TYPES = {
     "112": "API Credential",
@@ -24,12 +24,16 @@ TYPES = {
     "wallet.computer.Router": "Wireless Router",
 }
 
+ANSI_RED = u"\u001b[1;31m"
+ANSI_RESET = u"\u001b[0m"
 
 class OnepifEntry():
 
     def __init__(self, data):
         self.raw = data
         self.set_type(data["typeName"])
+        self.properties = []
+        self.parse()
 
     def set_type(self, new_type: str):
         if new_type not in TYPES:
@@ -65,6 +69,27 @@ class OnepifEntry():
             return self.raw["trashed"]
         return False
 
+    def add_property(self, property: oep.OnepifEntryProperty):
+        self.properties.append(property)
+
+    def get_property_keys(self):
+        keys = []
+        for p in self.properties:
+            keys.append(p.name)
+        keys = list(set(keys))
+        return keys
+
+    def get_property(self, key: str):
+        props = []
+        for p in self.properties:
+            if p.name == key:
+                props.append(p)
+        if not props:
+            return None
+        elif len(props) > 1:
+            print("{}Warning: Multiple properties matching '{}' found: {}. Ignoring all but the first.{}".format(ANSI_RED, key, repr(props), ANSI_RESET))
+        return props[0]
+
     def add_with_unique_key(self, prop_dict: dict, new_key: str, new_value):
         suffix_ctr = 0
         tmp_key = new_key
@@ -76,72 +101,29 @@ class OnepifEntry():
             new_value = new_value.replace("\x10", "")
         prop_dict[tmp_key] = new_value
 
-    def convert_section_field_to_string(self, field_data: dict) -> str:
+    def parse_section(self, section: dict):
-        kind = field_data["k"]
-        if kind in ["string", "concealed", "email", "phone", "URL", "menu", "cctype"]:
-            return field_data["v"]
-        elif kind == "date":
-            return datetime.fromtimestamp(field_data["v"]).strftime("%Y-%m-%d")
-        elif kind == "monthYear":
-            month = field_data["v"] % 100
-            month_name = datetime.strptime(str(month), "%m").strftime("%b")
-            year = field_data["v"] // 100
-            return "{} {}".format(month_name, year)
-        elif kind == "address":
-            addr = field_data["v"]
-            result = ""
-            if addr["street"]:
-                result += addr["street"] + "\n"
-            if addr["city"]:
-                result += addr["city"] + "\n"
-            if addr["zip"]:
-                result += addr["zip"] + "\n"
-            if addr["state"]:
-                result += addr["state"] + "\n"
-            if addr["region"]:
-                result += addr["region"] + "\n"
-            if addr["country"]:
-                result += addr["country"].upper()
-            return result.strip()
-        elif kind == "reference":
-            print("WARNING: Links between items are not supported (entry: {} -> {}).".format(self.raw["title"], field_data["t"]), file=sys.stderr)
-            return field_data["t"]
-
-        raise Exception("Unknown data kind in section fields: {}".format(kind))
-        return field_data["v"]
-
-    def parse_section_into_dict(self, target_dict: dict, section: dict):
         sect_title = section["title"]
         for f in section["fields"]:
             if "v" not in f:
                 # Skip fields without data
                 continue
-            propname = "{}: {}".format(sect_title, f["t"].title())
+            prop = oep.OnepifEntryProperty.from_sectionfield(f, sect_title)
-            if not sect_title:
+            self.add_property(prop)
-                propname = f["t"]
-            propval = self.convert_section_field_to_string(f)
-            self.add_with_unique_key(target_dict, propname, propval)
 
-    def parse_fields_into_dict(self, target_dict: dict, fields: list):
+    def parse_fields(self, fields: list):
         for f in fields:
-            if f["type"] in ["C", "R"]:
+            prop = oep.OnepifEntryProperty.from_webfield(f)
-                # Skip unsupported fields
+            if prop:
-                print("Ignoring checkbox/radiobuttons value in entry {}.".format(self.raw["title"]), file=sys.stderr)
+                self.add_property(prop)
-                continue
-            if "value" not in f:
-                # Skip fields without data
-                continue
-            if "designation" in f:
-                propname = f["designation"]
-            else:
-                propname = f["name"]
-            propval = f["value"]
-            if f["type"] not in ["T", "P", "E"]:
-                raise Exception("Unknown field type discovered: {}".format(f["type"]))
-            self.add_with_unique_key(target_dict, propname, propval)
 
-    def get_all_props(self) -> dict:
+    def add_simple_prop(self, key: str, value):
-        props = {}
+        if value == "\x10":
+            # this seems to be an "empty" indicator, so skip this
+            return False
+        prop = oep.OnepifEntryProperty(key, value)
+        self.add_property(prop)
+
+    def parse(self):
         for k, v in self.raw.items():
             if k in ["openContents", "secureContents"]:
                 # handle open/secure groups of properties
@@ -157,16 +139,15 @@ class OnepifEntry():
                             if "fields" not in s:
                                 # Skip empty sections
                                 continue
-                            self.parse_section_into_dict(props, s)
+                            self.parse_section(s)
                         continue
                     elif k2 == "fields":
                         # For some reason this differs from the "fields" in a section
-                        self.parse_fields_into_dict(props, v2)
+                        self.parse_fields(v2)
                         continue
                     # Handle all other values (most probably string or int)
-                    self.add_with_unique_key(props, k2, v2)
+                    self.add_simple_prop(k2, v2)
                 continue
             # Handle all other values
-            self.add_with_unique_key(props, k, v)
+            self.add_simple_prop(k, v)
         # TODO: Maybe walk all keys and see if there's (xxx_dd), xxx_mm, xxx_yy and turn them into a date
-        return props

onepif/OnepifEntryProperty.py

@@ -0,0 +1,96 @@
+import sys
+from datetime import datetime
+
+
+class OnepifEntryProperty():
+
+    def __init__(self, name, raw_value):
+        self.name = name   # internal name
+        self.title = name  # user visible name
+        self.raw = raw_value
+        self.set_value(raw_value)
+
+        self.section = None
+        self.type = ""
+        self.web_field_name = None   # designation
+        self.is_web_field = False    # has web_field_name
+        self.is_protected = False
+
+    def __repr__(self):
+        return "<OnepifEntryProperty \"{}\" ({}) = {}>".format(self.name, self.title, repr(self.raw))
+
+    def set_value(self, new_value):
+        if new_value == "\x10":
+            self.value = ""
+        else:
+            self.value = str(new_value)
+
+    @classmethod
+    def from_sectionfield(cls, field_dict: dict, sect_title: str = None):
+        key = field_dict["t"]
+        if not key:
+            key = field_dict["n"]
+        p = cls(key, field_dict)
+        if sect_title:
+            p.section = sect_title
+            p.title = "{}: {}".format(sect_title, field_dict["t"].title())
+            p.name = "{}_{}".format(sect_title.lower(), field_dict["t"].lower())
+
+        kind = field_dict["k"]
+        if kind in ["string", "email", "phone", "URL", "menu", "cctype"]:
+            p.set_value(field_dict["v"])
+        elif kind == "concealed":
+            p.set_value(field_dict["v"])
+            p.is_protected = True
+        elif kind == "date":
+            p.set_value(datetime.fromtimestamp(field_dict["v"]).strftime("%Y-%m-%d"))
+        elif kind == "monthYear":
+            month = field_dict["v"] % 100
+            month_name = datetime.strptime(str(month), "%m").strftime("%b")
+            year = field_dict["v"] // 100
+            p.set_value("{} {}".format(month_name, year))
+        elif kind == "address":
+            addr = field_dict["v"]
+            result = ""
+            if addr["street"]:
+                result += addr["street"] + "\n"
+            if addr["city"]:
+                result += addr["city"] + "\n"
+            if addr["zip"]:
+                result += addr["zip"] + "\n"
+            if addr["state"]:
+                result += addr["state"] + "\n"
+            if addr["region"]:
+                result += addr["region"] + "\n"
+            if addr["country"]:
+                result += addr["country"].upper()
+            p.set_value(result.strip())
+        elif kind == "reference":
+            print("WARNING: Links between items are not supported (-> {}).".format(field_dict["t"]), file=sys.stderr)
+            p.set_value(field_dict["t"])
+        else:
+            raise Exception("Unknown data kind in section fields: {}".format(kind))
+
+        return p
+
+    @classmethod
+    def from_webfield(cls, field_dict: dict):
+        if field_dict["type"] in ["C", "R"]:
+            # Skip unsupported fields
+            print("WARNING: Ignoring checkbox/radiobuttons value in entry.".format(), file=sys.stderr)
+            return None
+        if "value" not in field_dict or not field_dict["value"]:
+            # Skip fields without data
+            return None
+        if "designation" in field_dict and field_dict["designation"]:
+            key = field_dict["designation"]
+        else:
+            key = field_dict["name"]
+        p = cls(key, field_dict)
+        p.is_web_field = True
+        if "id" in field_dict:
+            p.web_field_name = field_dict["id"]
+        p.set_value(field_dict["value"])
+        if field_dict["type"] not in ["T", "P", "E"]:
+            raise Exception("Unknown field type discovered: {}".format(field_dict["type"]))
+        return p