smem_token_parser.py

import sys
from os import access, R_OK
from os.path import isfile
from collections import defaultdict
import re

from smem_obj import SMEM_Obj, ObjType


def clean_smem_string_token(s):
    return str(s).replace("|","")

def removeComments(s):
    """ Trim any '#' content from the given string
    """
    ind = str(s).find('#')
    if (ind == -1):
        return s
    return s[:ind]

def read_tokens_from_lines(file):
    retval = []             # A list of all tokens from the "smem --add {}" contents.
    inSmemAdd = False       # Whether the read is inside an "smem --add {}" command

    try:
        for line in file:
            sline = line.lstrip()
            # Don't use comments
            if sline.startswith('#'):
                continue
            # Check if we're in an smem --add command
            if not inSmemAdd:
                if sline.startswith("smem --add {"):
                    inSmemAdd = True
                    # Only add from this line any content after the opening '{'
                    sline = sline[12:]
                else:
                    continue
            
            # Get the tokens: split on whitespace, pipe quotes, parentheses, and brackets, but only exclude whitespace delimiters and pipe quotes
            regex_pattern = '[\s+]|\\|(.+?)\\||(?<=\\))|(?<=\\()|(?<=\\{)|(?<=\\})'
            tokens = re.split(regex_pattern, removeComments(sline).rstrip())

            while None in tokens:
                tokens.remove(None)
            while '' in tokens:
                tokens.remove('')

            # Check for the closing character
            try:
                ind = tokens.index('}')
                # If no exception, it was found
                tokens = tokens[:ind]
                inSmemAdd = False
            except:
                pass

            # Add this line to the return list
            if len(tokens) > 0:
                retval.extend(tokens)
    except Exception as e:
        print("ERROR extracting tokens from the given file: "+str(e), file=sys.stderr)
        return None
    
    return retval

""" 
This method scans a file that holds an 'smem --add{}' command and returns the relevant tokens from that file
"""
def get_smem_tokens_from_local_file(filename):
    # Error check for reading the file
    if not isfile(filename):
        print("ERROR in get_smem_tokens_from_local_file(): File does not exist: '"+str(filename)+"'.", file=sys.stderr)
        return None
    if not access(filename, R_OK):
        print("ERROR in get_smem_tokens_from_local_file(): File is not readable: '"+str(filename)+"'.", file=sys.stderr)
        return None

    # Get the file content
    with open(filename) as file:
        retval = read_tokens_from_lines(file)

    # All done
    return retval


class SMEM_Parser():
    
    def __init__(self):
        # Init the data structured needed to parse the smem file
        self.smem_var_obj_map = defaultdict(SMEM_Obj)

    def parse_file(self, smem_tokens):
        # Iterate through the tokens
        current_obj = None
        current_attr = None
        isNextWMEId = False
        for token in smem_tokens:
            # Skip empty tokens
            if len(token) == 0:
                continue
            # Get WME ID
            if isNextWMEId:
                current_obj = self.smem_var_obj_map[token]
                current_obj.set_id_var(token)
                isNextWMEId = False
                continue
            # Get start of obj
            if token == '(':
                isNextWMEId = True
                continue
            # Get end of obj
            if token == ')':
                current_obj = None
                current_attr = None
                continue
            # Get attributes
            if token.startswith('^'):
                current_attr = token
                continue
            # Get values
            if current_obj != None and current_attr != None:
                # Add the WME to the current object
                if token.startswith('<'):
                    token_val = self.smem_var_obj_map[token]
                    token_val.set_id_var(token)
                else:
                    token_val = clean_smem_string_token(token)
                current_obj.add_wme(current_attr, token_val)
            else:
                print("ERROR: Unexpected token '"+token+"'.")
                return

    def get_context_root(self):
        for var,obj in self.smem_var_obj_map.items():
            for (attr,val) in obj.wme_list:
                if attr == "^context-root":
                    obj.obj_type = ObjType.CONTEXT
                    return obj
        return None