Re: [RFC][PATCH] utils: add new tool: Poke preprocessor

[Jose E. Marchesi]

Hi Mohammad.

I am a little confused about what this is :)

It is a template system right?  Like, stuff between {* ... *} will be
interpreted and compiled as Poke statements, and stuff between (* ... *)
will be interpreted and compiled as a Poke expression, and substituted
by the result value, right?

If so, I like the idea.  You can evolve the template system to make it
easy to generate complex reports on binary data.  Nifty :)

We use the utils/ directory for Poke-written utilities.  Something like
pk-expr (which I would probably call pokefmt, similar to recfmt) would
probably be better to live in its own subdirectory pokefmt.  It is no
more no less than another libpoke using application, much like the poke
CLI itself..

> 2023-09-18  Mohammad-Reza Nabipoor  <mnabipoor@gnu.org>
>
>       * utils/pk-expr.c: New Poke preprocessor.
>       * utils/Makefile.am (pk-expr): Add rules for new tool.
> ---
>
> Hi Jose.
>
> This is a useful tool for testing other tools (e.g., disasm/asm).
>
> As as example this tool will transform the following text:
>
> ```
>
> # {*
> # *}
> # (**)
>
> %{
> fun foo = (int<32> i) void:
> {
>   if (1)
>     {{{{
>       printf "= %i32d\n", i;
>     }}}}
> }
> var a = 1, b = 2;
> }%
>
> hi%{var x = 1;}%
>
> %{
> foo (1);
> {{{}}}}%
>
> "%(a)% + %(b)% %{foo (a+b);}%"
>
> %{
> var a = [11,2];
> a[1] = 1;
> printf "%v", x;
> }%
> ```
>
> into
>
> ```
>
> # {*
> # *}
> # (**)
>
>
>
> hi
>
> = 1
>
>
> "1 + 2 = 3
> "
>
> 1
> ```
>
> I chose `%{' and `}%' pair for code blocks, and `%(' and `)%' for Poke
> expression; other options are Pascal like `{*' and `*}' pair for code block;
> `(*' and `*)' for Poke expressions.  WDYT?
>
>
> If you like this approach, I can complete this program and add doc and
> tests.
> Any feedback is appreciated.
>
>
>
> Regards,
> Mohammad-Reza
>
>
>  ChangeLog         |   5 +
>  utils/Makefile.am |  10 ++
>  utils/pk-expr.c   | 381 ++++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 396 insertions(+)
>  create mode 100644 utils/pk-expr.c
>
> diff --git a/ChangeLog b/ChangeLog
> index bdccbfae..600a4b1a 100644
> --- a/ChangeLog
> +++ b/ChangeLog
> @@ -1,3 +1,8 @@
> +2023-09-18  Mohammad-Reza Nabipoor  <mnabipoor@gnu.org>
> +
> +     * utils/pk-expr.c: New Poke preprocessor.
> +     * utils/Makefile.am (pk-expr): Add rules for new tool.
> +
>  2023-09-16  Jose E. Marchesi  <jose.marchesi@oracle.com>
>  
>       * pickles/Makefile.am (dist_pickles_DATA): Add gpt.pk.
> diff --git a/utils/Makefile.am b/utils/Makefile.am
> index 498f45b4..a858ba4e 100644
> --- a/utils/Makefile.am
> +++ b/utils/Makefile.am
> @@ -32,3 +32,13 @@ pk-strings: pk-strings.in Makefile
>  pk-bin2poke: pk-bin2poke.in Makefile
>       $(do_subst) < $(srcdir)/pk-bin2poke.in > pk-bin2poke
>       chmod +x pk-bin2poke
> +
> +bin_PROGRAMS = pk-expr
> +pk_expr_SOURCES = pk-expr.c
> +
> +pk_expr_CPPFLAGS = -I$(top_builddir)/gl -I$(top_srcdir)/gl \
> +                   -I$(top_srcdir)/libpoke -I$(top_builddir)/libpoke
> +pk_expr_CFLAGS = -Wall
> +pk_expr_LDADD  = $(top_builddir)/gl/libgnu.la \
> +                 $(top_builddir)/libpoke/libpoke.la
> +pk_expr_LDFLAGS =
> diff --git a/utils/pk-expr.c b/utils/pk-expr.c
> new file mode 100644
> index 00000000..b226fcff
> --- /dev/null
> +++ b/utils/pk-expr.c
> @@ -0,0 +1,381 @@
> +
> +/* Description:
> +     Poke pre-processor reads template from standard input, and execute
> +     Poke statements and expression and write the result to the standard
> +     output.
> +     Text between `%{' and `}%' pair is considered as Poke script and
> +     text between `%(' and `)%' pair should be valid Poke expression.  */
> +
> +/* TODO track line number and column.  */
> +
> +#include <config.h>
> +
> +#include <assert.h>
> +#include <err.h>
> +#include <inttypes.h>
> +#include <limits.h>
> +#include <string.h>
> +
> +#include "libpoke.h"
> +#include "read-file.h"
> +
> +enum poke_section_type
> +{
> +  POKE_SECTION_STMT,
> +  POKE_SECTION_EXPR,
> +};
> +
> +static const char *
> +poke_section_delim (enum poke_section_type type, int openning_p)
> +{
> +  return type == POKE_SECTION_STMT ? (openning_p ? "%{" : "}%")
> +                                   : (openning_p ? "%(" : ")%");
> +}
> +
> +static int
> +poke_section_delim_len (enum poke_section_type type)
> +{
> +  return 2;
> +}
> +
> +#define POKE_STMT_DELIM1 (poke_section_delim (POKE_SECTION_STMT, 1))
> +#define POKE_STMT_DELIM2 (poke_section_delim (POKE_SECTION_STMT, 0))
> +#define POKE_EXPR_DELIM1 (poke_section_delim (POKE_SECTION_EXPR, 1))
> +#define POKE_EXPR_DELIM2 (poke_section_delim (POKE_SECTION_EXPR, 0))
> +
> +struct poke
> +{
> +  pk_compiler compiler;
> +  FILE *output;
> +};
> +
> +void poke_init (struct poke *, const char *poke_src_file, FILE *output);
> +void poke_free (struct poke *);
> +
> +static const char *consume_poke_section (enum poke_section_type, struct poke 
> *,
> +                                         char **content_ptr, int 
> *content_len,
> +                                         char *start);
> +
> +int
> +main ()
> +{
> +  FILE *input = stdin;
> +  FILE *output = stdout;
> +  char *content, *content_original;
> +  int len;
> +  char *begin_stmt, *begin_expr;
> +  struct poke poke;
> +
> +  {
> +    size_t content_len;
> +
> +    content = fread_file (input, RF_BINARY, &content_len);
> +    if (content == NULL)
> +      err (1, "fread_file () failed");
> +    if (content_len == 0)
> +      return 0;
> +    if (content_len > (unsigned)INT_MAX)
> +      errx (1, "input file too large");
> +
> +    len = (int)content_len;
> +    content_original = content;
> +  }
> +
> +  poke_init (&poke, /*FIXME*/ NULL, output);
> +
> +beginning:
> +
> +  begin_stmt = strstr (content, POKE_STMT_DELIM1);
> +  begin_expr = strstr (content, POKE_EXPR_DELIM1);
> +
> +  if (begin_stmt == NULL)
> +    {
> +      /* There's no Poke statement.  Just search for Poke expressions.  */
> +
> +      if (begin_expr == NULL)
> +        {
> +          /* There's also no Poke expressions.  Dump the whole content.  */
> +          fprintf (output, "%.*s", len, content);
> +          return 0;
> +        }
> +
> +      /* Search for Poke expressions.  */
> +      while (1)
> +        {
> +          consume_poke_section (POKE_SECTION_EXPR, &poke, &content, &len,
> +                                begin_expr);
> +
> +          begin_expr = strstr (content, POKE_EXPR_DELIM1);
> +          if (begin_expr == NULL)
> +            {
> +              /* Print the content after last POKE_EXPR_DELIM1.  */
> +              fprintf (output, "%.*s", len, content);
> +              break;
> +            }
> +        }
> +    }
> +  else
> +    {
> +      /* There's at least one Poke statement.  */
> +
> +      if (begin_expr == NULL)
> +        {
> +          /* Search for Poke statements.  */
> +
> +          while (1)
> +            {
> +              consume_poke_section (POKE_SECTION_STMT, &poke, &content, &len,
> +                                    begin_stmt);
> +
> +              begin_stmt = strstr (content, POKE_STMT_DELIM1);
> +              if (begin_stmt == NULL)
> +                {
> +                  /* Print the content after last POKE_STMT_DELIM1.  */
> +                  fprintf (output, "%.*s", len, content);
> +                  break;
> +                }
> +            }
> +        }
> +      else
> +        {
> +          /* Search for both Poke statements and expressions whatever comes
> +             first.  */
> +
> +          if (begin_stmt < begin_expr)
> +            consume_poke_section (POKE_SECTION_STMT, &poke, &content, &len,
> +                                  begin_stmt);
> +          else
> +            consume_poke_section (POKE_SECTION_EXPR, &poke, &content, &len,
> +                                  begin_expr);
> +
> +          goto beginning;
> +        }
> +    }
> +
> +  poke_free (&poke);
> +  free (content_original);
> +  return 0;
> +}
> +
> +static void poke_stmt (struct poke *, char *begin, char *end);
> +
> +static void poke_expr (struct poke *, const char *begin, const char *end);
> +
> +static const char *
> +consume_poke_section (enum poke_section_type type, struct poke *pk,
> +                      char **content_ptr, int *content_len, char *begin)
> +{
> +  const char *content = *content_ptr;
> +  const char *delim1 = poke_section_delim (type, 1);
> +  const char *delim2 = poke_section_delim (type, 0);
> +  char *end;
> +
> +  assert (strncmp (begin, delim1, poke_section_delim_len (type)) == 0);
> +
> +  /* Print the content before the current section literally.  */
> +  fprintf (pk->output, "%.*s", (int)(uintptr_t)(begin - content), content);
> +
> +  begin += 2; /* Skip openning delimiter.  */
> +  end = strstr (begin, delim2);
> +  if (end == NULL)
> +    /* TODO Add source location info.  */
> +    errx (1, "Non-terminated Poke statement section: '%s' not found", 
> delim2);
> +
> +  if (type == POKE_SECTION_STMT)
> +    poke_stmt (pk, begin, end);
> +  else
> +    poke_expr (pk, begin, end);
> +
> +  *content_ptr = end + 2;
> +  *content_len -= end + 2 - content;
> +
> +  return end;
> +}
> +
> +//--- poke
> +
> +// terminal IO functions
> +static void
> +tif_flush (void)
> +{
> +  fflush (stdout);
> +}
> +static void
> +tif_puts (const char *s)
> +{
> +  printf ("%s", s);
> +}
> +static void
> +tif_printf (const char *fmt, ...)
> +{
> +  va_list ap;
> +
> +  va_start (ap, fmt);
> +  vprintf (fmt, ap);
> +  va_end (ap);
> +}
> +static void
> +tif_indent (unsigned int level, unsigned int step)
> +{
> +  putchar ('\n');
> +  for (unsigned int i = 0; i < step * level; ++i)
> +    putchar (' ');
> +}
> +static void
> +tif_class (const char *name)
> +{
> +  (void)name;
> +}
> +static int
> +tif_class_end (const char *name)
> +{
> +  (void)name;
> +  return 1;
> +}
> +static void
> +tif_hlink (const char *name, const char *id)
> +{
> +  (void)name;
> +  (void)id;
> +}
> +static int
> +tif_hlink_end (void)
> +{
> +  return 1;
> +}
> +static struct pk_color
> +tif_color (void)
> +{
> +  static struct pk_color c = {
> +    .red = 0,
> +    .green = 0,
> +    .blue = 0,
> +  };
> +  return c;
> +}
> +static struct pk_color
> +tif_bgcolor (void)
> +{
> +  static struct pk_color c = {
> +    .red = 255,
> +    .green = 255,
> +    .blue = 255,
> +  };
> +  return c;
> +}
> +static void
> +tif_color_set (struct pk_color c)
> +{
> +  (void)c;
> +}
> +static void
> +tif_bgcolor_set (struct pk_color c)
> +{
> +  (void)c;
> +}
> +
> +void
> +poke_init (struct poke *pk, const char *poke_src_file, FILE *output)
> +{
> +  static struct pk_term_if tif = {
> +    .flush_fn = tif_flush,
> +    .puts_fn = tif_puts,
> +    .printf_fn = tif_printf,
> +    .indent_fn = tif_indent,
> +    .class_fn = tif_class,
> +    .end_class_fn = tif_class_end,
> +    .hyperlink_fn = tif_hlink,
> +    .end_hyperlink_fn = tif_hlink_end,
> +    .get_color_fn = tif_color,
> +    .get_bgcolor_fn = tif_bgcolor,
> +    .set_color_fn = tif_color_set,
> +    .set_bgcolor_fn = tif_bgcolor_set,
> +  };
> +  int ret;
> +  pk_val pexc;
> +
> +  pk->output = output;
> +  pk->compiler = pk_compiler_new (&tif);
> +  if (pk->compiler == NULL)
> +    errx (1, "pk_compiler_new() failed");
> +
> +  if (poke_src_file)
> +    {
> +      ret = pk_compile_file (pk->compiler, poke_src_file, &pexc);
> +      if (ret != PK_OK)
> +        errx (1, "pk_compile_file() failed for user-provided file (%s)",
> +              poke_src_file);
> +      else if (pexc != PK_NULL)
> +        /* FIXME improve the error message.  */
> +        errx (1, "unhandled exception while running user-provided file (%s)",
> +              poke_src_file);
> +    }
> +}
> +
> +void
> +poke_free (struct poke *pk)
> +{
> +  pk_compiler_free (pk->compiler);
> +}
> +
> +static void
> +poke_stmt (struct poke *poke, char *begin, char *end)
> +{
> +  int res;
> +  char tmp;
> +  pk_val exc;
> +
> +  assert (end != NULL);
> +
> +  tmp = *end;
> +  *end = '\0';
> +  /* TODO use pk_compile_buffer_with_loc.  */
> +  res = pk_compile_buffer (poke->compiler, begin, NULL, &exc);
> +  *end = tmp;
> +
> +  if (res != PK_OK)
> +    /* TODO add location info.  */
> +    errx (1, "pk_compile_buffer() failed for Poke code: '%.*s'",
> +          (int)(uintptr_t)(end - begin), begin);
> +  else if (exc != PK_NULL)
> +    /* TODO improve the error message.  */
> +    errx (1, "unhandled exception happend during execution of '%.*s'",
> +          (int)(uintptr_t)(end - begin), begin);
> +}
> +
> +static void
> +poke_expr (struct poke *poke, const char *begin, const char *end)
> +{
> +  int res;
> +  pk_val val, exc;
> +  size_t n;
> +  char *code;
> +
> +  assert (end != NULL);
> +
> +  n = end - begin;
> +  code = malloc (n + 2);
> +  if (code == NULL)
> +    err (1, "malloc() failed");
> +
> +  memcpy (code, begin, n);
> +  code[n] = ';';
> +  code[n + 1] = '\0';
> +  /* TODO use pk_compile_statement_with_loc.  */
> +  res = pk_compile_statement (poke->compiler, code, NULL, &val, &exc);
> +
> +  if (res != PK_OK)
> +    /* TODO add location info.  */
> +    errx (1, "pk_compile_statement() failed for Poke expression: '%.*s'",
> +          (int)n, code);
> +  else if (exc != PK_NULL)
> +    /* TODO improve the error message.  */
> +    errx (1, "unhandled exception happend during execution of '%.*s'", 
> (int)n,
> +          code);
> +  else if (val == PK_NULL)
> +    errx (1, "expected a Poke expression, got a Poke statement in '%.*s'",
> +          (int)n, code);
> +
> +  free (code);
> +  pk_print_val (poke->compiler, val, &exc);
> +}