#
# This software is Copyright 2005 by Elsevier Inc.  You may use it
# under the terms of the license at http://perl.plover.com/hop/LICENSE.txt .
#



###
### tokens-1
###

## Chapter 8 section 1.3

sub tokens {
  my ($input, $label, $pattern) = @_;
  my @tokens;
  my ($buf, $finished) = ("");
  my $split = sub { split /($pattern)/, $_[0] };
  my $maketoken = sub { [$label, $_[0] ]};
  sub {
    while (@tokens == 0 && ! $finished) {
      my $i = $input->();
      if (ref $i) {          # Input has already been tokenized
        my ($sep, $tok) = $split->($buf);
        $tok = $maketoken->($tok) if defined $tok;
        push @tokens, grep $_ ne "", $sep, $tok, $i;
        $buf = "";
      } else {               # Input is an untokenized string
        $buf .= $i if defined $i; # Append new input to buffer
        my @newtoks = $split->($buf);
        while (@newtoks > 2 
               || @newtoks && ! defined $i) {
          # Buffer contains complete separator plus complete token
          # OR we've reached the end of the input
          push @tokens, shift(@newtoks);
          push @tokens, $maketoken->(shift @newtoks) if @newtoks;
        }
        # Reassemble remaining contents of buffer
        $buf = join "", @newtoks;
        $finished = 1 if ! defined $i;
        @tokens = grep $_ ne "", @tokens;
      }
    }
    return shift(@tokens);
  }
}