PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Growing Languages with Metamorphic Syntax Macros Claus Brabrand Michael Schwartzbach BRICS, University of Aarhus, Denmark

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Outline Introduction Metamorphisms vDSL Specificity parsing Related and future work Conclusion

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Lexical Macros M LEX : (TOKENS) n  TOKENS, n  0

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Lexical Macros M LEX : (TOKENS) n  TOKENS, n  0 #define square(X) X*X

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Lexical Macros M LEX : (TOKENS) n  TOKENS, n  0 #define square(X) X*X square(y+1)  y+1*y+1 #define square(X) X*X square(y+1)  y+1*y+1

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Lexical Macros M LEX : (TOKENS) n  TOKENS, n  0  #define square(X) X*X square(y+1)  y+1*y+1 #define square(X) X*X square(y+1)  y+1*y+1

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Lexical Macros M LEX : (TOKENS) n  TOKENS, n  0  #define square(X) X*X square(y+1)  y+1*y+1 #define square(X) X*X square(y+1)  y+1*y+1 #define square(X) (X)*(X) square(y+1)  (y+1)*(y+1) #define square(X) (X)*(X) square(y+1)  (y+1)*(y+1)

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Lexical Macros M LEX : (TOKENS) n  TOKENS, n  0  Problem: Independent of syntax! Unsafe: parse errors discovered at invocation-time #define square(X) X*X square(y+1)  y+1*y+1 #define square(X) X*X square(y+1)  y+1*y+1 #define square(X) (X)*(X) square(y+1)  (y+1)*(y+1) #define square(X) (X)*(X) square(y+1)  (y+1)*(y+1)

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Invocation Syntax #define swap(X,Y) { int t=X; X=Y; Y=t; }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Invocation Syntax #define swap(X,Y) { int t=X; X=Y; Y=t; } if (a>b) swap(a,b); else b=0; if (a>b) swap(a,b); else b=0; *** parse error!

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Invocation Syntax #define swap(X,Y) { int t=X; X=Y; Y=t; } if (a>b) swap(a,b); else b=0; if (a>b) swap(a,b); else b=0; *** parse error! 

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Invocation Syntax #define swap(X,Y) { int t=X; X=Y; Y=t; } if (a>b) swap(a,b); else b=0; if (a>b) swap(a,b); else b=0; *** parse error! #define swap(X,Y) do { int t=X; X=Y; Y=t; } while (0) if (a>b) swap(a,b); else b=0; if (a>b) swap(a,b); else b=0; 

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Invocation Syntax Problem: fixed invocation syntax! same for exp / stm / … #define swap(X,Y) { int t=X; X=Y; Y=t; } if (a>b) swap(a,b); else b=0; if (a>b) swap(a,b); else b=0; *** parse error! #define swap(X,Y) do { int t=X; X=Y; Y=t; } while (0) if (a>b) swap(a,b); else b=0; if (a>b) swap(a,b); else b=0;  M(x,y,z)

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Macro M SYN : (AST) n  AST, n  0 Typed with nonterminals of the host grammar Safe: no parse errors as a conseq. of expansion

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Macro M SYN : (AST) n  AST, n  0 Typed with nonterminals of the host grammar Safe: no parse errors as a conseq. of expansion stm  repeat stm until ( exp ) ;

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Macro M SYN : (AST) n  AST, n  0 Typed with nonterminals of the host grammar Safe: no parse errors as a conseq. of expansion macro  repeat until ( ) ; ::= { { bool first = true; while (first || ! ) { first = false; } } } macro  repeat until ( ) ; ::= { { bool first = true; while (first || ! ) { first = false; } } } stm  repeat stm until ( exp ) ;

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Macro M SYN : (AST) n  AST, n  0 Typed with nonterminals of the host grammar Safe: no parse errors as a conseq. of expansion macro  repeat until ( ) ; ::= { { bool first = true; while (first || ! ) { first = false; } } } macro  repeat until ( ) ; ::= { { bool first = true; while (first || ! ) { first = false; } } } 1.Invocation syntax: grammar extension stm  repeat stm until ( exp ) ;

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Macro M SYN : (AST) n  AST, n  0 Typed with nonterminals of the host grammar Safe: no parse errors as a conseq. of expansion macro  repeat until ( ) ; ::= { { bool first = true; while (first || ! ) { first = false; } } } macro  repeat until ( ) ; ::= { { bool first = true; while (first || ! ) { first = false; } } } 1.Invocation syntax: grammar extension 2.Transformation: morphing into host syntax stm  repeat stm until ( exp ) ;

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility? macro  enum { }; ::= {…} macro  enum {, }; ::= {…} macro  enum {,, }; ::= {…} macro  enum { }; ::= {…} macro  enum {, }; ::= {…} macro  enum {,, }; ::= {…}

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility? macro  enum { }; ::= {…} macro  enum {, }; ::= {…} macro  enum {,, }; ::= {…} macro  enum { }; ::= {…} macro  enum {, }; ::= {…} macro  enum {,, }; ::= {…} decls  enum { id } ;  enum { id, id } ;  enum { id, id, id } ; decls  enum { id } ;  enum { id, id } ;  enum { id, id, id } ;

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility? macro  enum { }; ::= {…} macro  enum {, }; ::= {…} macro  enum {,, }; ::= {…} macro  enum { }; ::= {…} macro  enum {, }; ::= {…} macro  enum {,, }; ::= {…} decls  enum { id } ;  enum { id, id } ;  enum { id, id, id } ; decls  enum { id } ;  enum { id, id } ;  enum { id, id, id } ; Problems: Only fixed (finite) arity Highly redundant

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility [Scheme]: special list constructor: “...”

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility [Scheme]: special list constructor: “...” decls  ( enum id * )

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility [Scheme]: special list constructor: “...” [MS 2 ]: lists +/ , options ?, tuples {…}, and token-separated lists  T decls  ( enum id * )

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility [Scheme]: special list constructor: “...” [MS 2 ]: lists +/ , options ?, tuples {…}, and token-separated lists  T decls  ( enum id * ) decls  enum { id , } ; ~ E-BNF

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility! Allow user-defined nonterminals (in invocation syntax):

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility! Allow user-defined nonterminals (in invocation syntax): decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Flexibility! Allow user-defined nonterminals (in invocation syntax): Transformation? without compromising safety decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorphisms Our solution: Metamorphisms Attach host nonterminals to a user-def’d nonterminal metamorph m();

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorphisms Our solution: Metamorphisms Attach host nonterminals to a user-def’d nonterminal metamorph m(); macro  … … ::= { … … } metamorph m(); macro  … … ::= { … … }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorphisms Our solution: Metamorphisms Attach host nonterminals to a user-def’d nonterminal Specify morphing (into host syntax) inductively metamorph m(); macro  … … ::= { … … } metamorph m(); macro  … … ::= { … … }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorphisms Our solution: Metamorphisms Attach host nonterminals to a user-def’d nonterminal Specify morphing (into host syntax) inductively metamorph m(); macro  … … ::= { … … } morph  … ::= { … } metamorph m(); macro  … … ::= { … … } morph  … ::= { … }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorphisms Our solution: Metamorphisms Attach host nonterminals to a user-def’d nonterminal Specify morphing (into host syntax) inductively Non-local transformations (multiple results) metamorph m(); macro  … … ::= { … … } morph  … ::= { … } metamorph m(); macro  … … ::= { … … } morph  … ::= { … }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorphisms Our solution: Metamorphisms Attach host nonterminals to a user-def’d nonterminal Specify morphing (into host syntax) inductively Non-local transformations (multiple results) metamorph m(); macro  … … ::= { … … … } morph  … ::= { … } { … } metamorph m(); macro  … … ::= { … … … } morph  … ::= { … } { … }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Example: enum decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Example: enum decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums   enum { x, y, z }; const int x = 0; const int y = 1; const int z = 2; const int x = 0; const int y = 1; const int z = 2; 

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Example: enum decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums   enum { x, y, z }; const int x = 0; const int y = 1; const int z = 2; const int x = 0; const int y = 1; const int z = 2;  Without compile-time programming language (with AST values)

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Example: enum decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums   enum { x, y, z }; int e = 0; const int x = e++; const int y = e++; const int z = e++; int e = 0; const int x = e++; const int y = e++; const int z = e++;  Without compile-time programming language (with AST values)

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Example: enum metamorph enums(); decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Example: enum metamorph enums(); macro  enum { } ; ::= { int e = 0; const int = e++; } metamorph enums(); macro  enum { } ; ::= { int e = 0; const int = e++; } decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Example: enum metamorph enums(); macro  enum { } ; ::= { int e = 0; const int = e++; } morph , ::= { const int = e++; } metamorph enums(); macro  enum { } ; ::= { int e = 0; const int = e++; } morph , ::= { const int = e++; } decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Example: enum metamorph enums(); macro  enum { } ; ::= { int e = 0; const int = e++; } morph , ::= { const int = e++; } morph  ::= { } metamorph enums(); macro  enum { } ; ::= { int e = 0; const int = e++; } morph , ::= { const int = e++; } morph  ::= { } decls  enum { id enums } ; enums , id enums   decls  enum { id enums } ; enums , id enums  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 stm  reserve ( res ) stm res  id res   stm  reserve ( res ) stm res  id res   Metamorph Example: reserve reserve ( a b c )...; acquire(a); acquire(b); acquire(c);...; release(c); release(b); release(a); acquire(a); acquire(b); acquire(c);...; release(c); release(b); release(a); 

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Multiple Results Example: reserve metamorph res(); stm  reserve ( res ) stm res  id res   stm  reserve ( res ) stm res  id res  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Multiple Results Example: reserve metamorph res(); macro  reserve ( ) ::= { { } } metamorph res(); macro  reserve ( ) ::= { { } } stm  reserve ( res ) stm res  id res   stm  reserve ( res ) stm res  id res  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Multiple Results Example: reserve metamorph res(); macro  reserve ( ) ::= { { } } morph  ::= { acquire( ); } { release( ); } metamorph res(); macro  reserve ( ) ::= { { } } morph  ::= { acquire( ); } { release( ); } stm  reserve ( res ) stm res  id res   stm  reserve ( res ) stm res  id res  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Multiple Results Example: reserve metamorph res(); macro  reserve ( ) ::= { { } } morph  ::= { acquire( ); } { release( ); } morph  ::= { } { } metamorph res(); macro  reserve ( ) ::= { { } } morph  ::= { acquire( ); } { release( ); } morph  ::= { } { } stm  reserve ( res ) stm res  id res   stm  reserve ( res ) stm res  id res  

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Advantages Flexibility Safety Simplicity

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Advantages Flexibility: Tree structures Non-local transformations (multiple results) Safety Simplicity

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Advantages Flexibility: Tree structures Non-local transformations (multiple results) Safety: No parse errors as a conseq. of macro expansion Guaranteed termination Simplicity

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Advantages Flexibility: Tree structures Non-local transformations (multiple results) Safety: No parse errors as a conseq. of macro expansion Guaranteed termination Simplicity: Based entirely on declarative concepts: grammars and substitution

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 vDSL: very Domain Specific Language studies course Math101 title “Mathematics 101” 2 point fall term … exclusions Math101 <> MathA Math102 <> MathB prerequisites Math101, Math102 < Math201, Math202, Math203 Math101, CS101 < CS202 studies course Math101 title “Mathematics 101” 2 point fall term … exclusions Math101 <> MathA Math102 <> MathB prerequisites Math101, Math102 < Math201, Math202, Math203 Math101, CS101 < CS202

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Specificity Parsing macro  select from where macro  select all from where macro  select from where macro  select all from where

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Specificity Parsing Challenge rounds: –Select most specific productions (wrt. FIRST sets) macro  select from where macro  select all from where macro  select from where macro  select all from where

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Specificity Parsing Challenge rounds: –Select most specific productions (wrt. FIRST sets) Resolves many ambiguities Independent of definition-order Overloading Avoids keywordification Commit  no branch explosion, no backtracking macro  select from where macro  select all from where macro  select from where macro  select all from where

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Related Work: Macro Survey 8x Macro languages: { CPP, M4, T E X, Dylan, C++ Templates, Scheme, JTS, MS 2 }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Related Work: Macro Survey 8x Macro languages: { CPP, M4, T E X, Dylan, C++ Templates, Scheme, JTS, MS 2 } 31x Macro properties: { Level of operation, Programmability, Definition scope, Termination, Argument syntax, Error trailing, … }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Related Work: Macro Survey 8x Macro languages: { CPP, M4, T E X, Dylan, C++ Templates, Scheme, JTS, MS 2 } 31x Macro properties: { Level of operation, Programmability, Definition scope, Termination, Argument syntax, Error trailing, … }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Related Work “Extensible Syntax with Lexical Scoping” - by Cardelli, Matthes, and Abadi : Not a macro language, but a parser generator: –Target language (not host language) –Extend  recompile parser Localized transformation only Disjoint productions Keywordification Explicit alpha conversion

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Future Work: metafront Extensible syntax processor, based on: Specificity parsing and Metamorphic syntax macros: metafront base grammar: L macros: L+  L program in L+ program in L

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Future Work: metafront Extensible syntax processor, based on: Specificity parsing and Metamorphic syntax macros: Safe transformation: L+  L metafront base grammar: L macros: L+  L program in L+ program in L

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Future Work: metafront Extensible syntax processor, based on: Specificity parsing and Metamorphic syntax macros: Safe transformation: L+  L Safe transformation: L’  L, (vDSL) metafront base grammar: L macros: L+  L program in L+ program in L

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Conclusion Metamorphic Syntax Macros is a… flexible safe simple …way of Growing Languages

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Conclusion Metamorphic Syntax Macros is a… flexible safe simple …way of Growing Languages Fully implemented in (language for developing interactive Web services)

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 The End next: bonus slides

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Pretty Printing & Error Reporting Pretty Printing: Terminal printers: ASCII, L A E X, HTML ( +/- expansion) Error Reporting: stdout, HTML *** symbol errors: *** test.wig:7: Identifier ‘inf’ not declared in macro argument ‘S’ in macro invocation ‘reader’ (test.wig:7) defined in [std.wigmac:44] *** symbol errors: *** test.wig:7: Identifier ‘inf’ not declared in macro argument ‘S’ in macro invocation ‘reader’ (test.wig:7) defined in [std.wigmac:44] T

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002  -Conversion macro  repeat until ( ) ; ::= { { bool first = true; while (first || ! ) { first = false; } macro  repeat until ( ) ; ::= { { bool first = true; while (first || ! ) { first = false; }

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002  -Conversion metamorph enums(); macro  enum { } ; ::= { int e = 0; const int = e++; } morph , ::= { const int = e++; } morph  ::= { } metamorph enums(); macro  enum { } ; ::= { int e = 0; const int = e++; } morph , ::= { const int = e++; } morph  ::= { } decls: enum { id enums } ; enums:, id enums |  decls: enum { id enums } ; enums:, id enums | 

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Example: switch metamorph swb(); macro  switch ( ) { } ::= { { var x = ; } morph  case : break; ::= { if (x== ) { } else } morph  case : break; ::= { if (x== ) { } } metamorph swb(); macro  switch ( ) { } ::= { { var x = ; } morph  case : break; ::= { if (x== ) { } else } morph  case : break; ::= { if (x== ) { } } stm: switch ( exp ) { swb } swb: case exp : stms break; swb | case exp : stms break; stm: switch ( exp ) { swb } swb: case exp : stms break; swb | case exp : stms break;

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Metamorph Wellformedness Check at definition time: “no left recursion” (guarantees termination):  “derivability” (metamorphisms must derive something finite) :  xlist  xlist X   xlist  xlist X   xlist  X xlist   xlist  X xlist   xlist  X xlist   xlist  X xlist   xlist  X xlist

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Representation macro  M XY ( ) ::= { X,, Y } A, M XY (B, C), D  A, X, B, C, Y, D macro  M XY ( ) ::= { X,, Y } A, M XY (B, C), D  A, X, B, C, Y, D “Weaving” yields transparency! weave

PEPM 2002 Growing Languages with Metamorphic Syntax Macros January 14, 2002 Syntax Macros M SYN : (AST) n  AST, n  0 ~ square( )  macro  square ( ) ::= { * } macro  square ( ) ::= { * } * * exp E exp E * * exp y + 1y + 1 y + 1y + 1 y + 1y + 1 y + 1y + 1 E exp E y + 1y + 1 y + 1y + 1 exp