
% cnv.tex -- macro for string conversion
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% April 2005                                         Petr Olsak


% \cnvin {<input text>} converts <input text> to <output text>.
% This output is stored in \cnvout macro.

% For more information see the end of this file.
% The comments after %% (double percent sign) are intended for macro 
% programmers. See "simple" documentation at the end of this file first.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%% we cannot read this macro twice:

\ifx\cnvin\undefined \else \endinput \fi  

%% \predef <token> defines macro \E:<token>:<table>
%% \findef <token> defines macro \T:<token>:<table>

\def\predef #1{\expandafter \predefmethod \csname E:\string#1:\cnvtable\endcsname}
\def\findef #1#2{\expandafter \findefmethod \csname T:\string#1:\cnvtable\endcsname {#2}}

%% \cnvaccent <command> <char> {<output>} does following:
%% 1. it defines \E:<command>:<table> #1 as {\runcnvaccent \A:<command>:#1:<table>}
%%    (\runcnvaccent implements the test if accented letter is undeclared, see below)
%% 2. it defines \A:<command>:<char>:<table> as {<output>}
%% 3. if "<command> <char>" is one <token> after expansion, then 
%%    it defines \T:<token>:<table> as {<output>}
       
\def\cnvaccent #1#2#3{\expandafter \cnvaccentmethod \csname E:\string#1:\cnvtable\endcsname ##1%
       {\cnvexpand \expandafter \runcnvaccent \csname A:\string#1:\string##1:\cnvtable\endcsname}%
   \expandafter \findefmethod \csname A:\string#1:\string#2:\cnvtable\endcsname {#3}%
   \edef \tmp{#1{#2}}%
   \expandafter \cnvtestaccent \tmp \cnvtestaccent
   \ifx\nextchar\empty \expandafter \findefmethod \csname T:\tmp:\cnvtable\endcsname {#3}\fi
}
\def\cnvtestaccent {\futurelet \nextchar \docnvtestaccent}
\def\docnvtestaccent #1#2\cnvtestaccent{\ifx\nextchar \bgroup
   \else \def\nextchar{#2}\fi}

%% main macro:

\long\def\cnvin #1{\def\cnvout{}\cnvnext #1\cnvend}

%% \cnvnext: we test the next token by \futurelet because it may be a space
%% or a brace:

\def\cnvnext {\futurelet \nextchar \docnvnext}
\def\docnvnext {\let\next=\cnvtoken
   \ifx \nextchar\cnvend        \let\next=\relax \fi
   \ifx \nextchar\spacetoken    \let\next=\cnvspace \fi   
   \ifx \nextchar\cnvbgroup     \let\next=\cnvopenbrace \fi
   \ifx \nextchar\cnvegroup     \let\next=\cnvclosebrace \fi
   \ifx \nextchar\runcnvaccent  \let\next=\relax \fi
   \ifx \nextchar\cnvexec       \let\next=\relax \fi
   \ifx \nextchar\cnvexpand     \let\next=\relax \fi
   \ifx \nextchar\cnvexpandtext \let\next=\relax \fi
   \next}

%% \cnvend: end of conversion. We define it as \outer because
%% the parameter of user macros cannot scan this token.

\outer\def\cnvend{\let\nextchar=\relax}

%% \cnvbgroup, \cnvegroup: We suppose that these sequences will have
%% the constant meaning (user may change the \bgroup, \egroup meaning).

\let\cnvbgroup={  \let\BGROUP={  \let\EGROUP=}  \let\cnvegroup=}

%% \spacetoken is the control sequence equal to space token (catcode 10):

{\def\\{\global\let\spacetoken= }\\ }

%% \cnvtoken: next <token> is scannable in #1 parameter
%% It expands to \E:<token>:<table> (if defined) else
%% it stores contents of \T:<token>:<table> to \cnvout (if defined) else
%% it stores <token> to \cnvout using \cnvdefault macro.

\def\cnvtoken #1{\def\tmpc{#1}\docnvtoken}
\def\docnvtoken{%   
   \expandafter \ifx \csname E:\expandafter\string\tmpc:\cnvtable\endcsname \relax 
      \expandafter \ifx \csname T:\expandafter\string\tmpc:\cnvtable\endcsname \relax 
         \ifx\tmpc\space \cnvstoree{\expandafter \cnvout \tmpc}%
         \else \cnvstoreee {\expandafter \expandafter \expandafter \cnvout
                            \expandafter \cnvdefault \tmpc}%
         \fi
      \else
         \cnvstoreee {\expandafter \expandafter \expandafter 
             \cnvout \csname T:\expandafter\string\tmpc:\cnvtable\endcsname}%
      \fi
      \expandafter \cnvnext
   \else
      \expandafter \expandafter \expandafter \cnvnext 
      \csname E:\expandafter\string\tmpc:\cnvtable\expandafter \endcsname
   \fi}

%% \cnvspace, \cnvopenbrace, \cnvclosebrace: the next token is unreadable
%% by #1 macro parameter. We define \tmpc, remove the next token
%% and run \docnvtoken macro.

\def\cnvspace      {\def\tmpc{ }\afterassignment       \docnvtoken \let\next= }
\def\cnvopenbrace  {\def\tmpc{\BGROUP}\afterassignment \docnvtoken \let\next= }
\def\cnvclosebrace {\def\tmpc{\EGROUP}\afterassignment \docnvtoken \let\next= }

%% \runcnvaccent: next token #1 is \A:<command>:<char>:<table>.
%% It stores the contents of \A:<command>:<char>:<table> to \cnvout
%% (if defined) else it stores \cnvadefault to \cnvout.

\def\runcnvaccent #1{\ifx #1\relax 
      \cnvstoree {\expandafter \cnvout \cnvadefault}%
   \else 
      \cnvstoree {\expandafter \cnvout #1}%
   \fi \cnvnext}

%% \cnvexec: we switch off the conversion until \cnvnext
%% The contents of this macro means: do nothing, but we need
%% somewhat special contents in order to distinguish it by \ifx.

\long\def\cnvexec {\empty\relax\relax}

%% \cnvexpand, \cnvexpandtext: see the documentation at the end of this file

\def\cnvexpand {\expandafter \cnvnext}
\def\cnvexpandtext #1{\edef\tmp{#1}\expandafter \cnvnext \tmp}

%% \cnvstoreee: macro appends the desired contens to \cnvout.
%% We cannot expand the \cnvout by \edef because the expandable tokens can be
%% included here. So we need to do some \expandafter gymnastics.

\def\cnvstore   {\expandafter \def \expandafter \cnvout \expandafter}
\def\cnvstoree  {\expandafter \cnvstore  \expandafter}
\def\cnvstoreee {\expandafter \cnvstoree \expandafter}

%% Implicit values:

\def\cnvcopy{}
\let\cnvdefault=\cnvcopy
\def\cnvadefault{?}
\def\cnvtable{}

\let\predefmethod=\gdef
\let\findefmethod=\gdef
\let\cnvaccentmethod=\gdef

\endinput

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Po proveden� p��kazu \cnvin {<seznam token�>}
je v makru \cnvout ukryt konvertovan� seznam token�.

After applying \cnvin{<list of tokens>} the converted list of tokens is
saved in \cnvout macro.

Konverze se prov�d� bez obvykl� expanze <seznamu token�> a je jedno, jak�
kategorie jednotliv� tokeny maj�. Konverze prob�h� ve dvou pr�chodech:

The conversion is done without the standard expansion of <list of tokens> 
and is independet of catcodes of converted tokens.
The conversion process is done in two steps:

1. Kontrolovan� ��ste�n� expanze jednotliv�ch token�. Po jej�m
   proveden� se konverze vrac� k v�sledku expanze a konvertuje jej
   znovu. Tokeny, kter� takto expanduj�, se v konverzn� tabulce
   deklaruj� pomoc� \predef. Tokeny, kter� nejsou takto deklarov�ny,
   p�i konverzi neexpanduj� (a�koli p�i b�n� expanzi t�eba expanduj�).
2. Fin�ln� konverze tokenu: token je konvertov�n podle deklarace
   makrem \findef na v�stupn� posloupnost token�, ke kter�m se u�
   konverze nevrac�.

1. Controlled partial expansion of tokens. The conversion process is
   returned back to the result of this expansion and converts it once
   more.  Only tokens declared by \predef in the conversion table are
   expanded this way, other tokens not defined this way are not
   expanded (even if these tokens are expandable in the normal \TeX{}
   meaning).
2. Final conversion of token: token is converted to the output string
   of tokens following to the declaration by \findef macro. The
   conversion process does not return to this output anymore.

Tokeny, kter� podl�haj� konverzi, mohou m�t libovolnou kategorii. Nen�
tedy nutn�, aby se jednalo o kontroln� sekvence.

The tokens subject to conversion can have an arbitrary catcode. Not only control
sequences can be converted.

Deklarace token� pomoc� \predef a \findef jsou zcela nez�visl� na
"skute�n�m v�znamu" tokenu. Jedn�-li se o kontroln� sekvenci, bude
se tato kontroln� sekvence p�i b�n�m zpracov�n� \TeX{}em chovat
b�n�m zp�sobem a�koli m� pro konverzi deklarov�n pomoc� \predef nebo
\findef odli�n� v�znam.

The declarations of tokens by \predef and \findef are quite
independent of the normal meaning of these tokens. It means that in case of
control sequences these sequences will behave during the usual processing
by \TeX{} in a usual way although for the purpose of conversion their meaning
can be changed using \predef or \findef.

Deklarace pomoc� \predef m��e m�t b�n� parametry a separ�tory (#1#2 atd.)

You can use normal parameters and separators (#1#2 etc.) when \predef
is used. 

Ka�d� token m��e m�t jen jeden v�znam deklarovan� pomoc� \predef nebo
\findef. Je-li token deklarov�n oboj�m zp�sobem, m� p�i konverzi p�ednost
\predef. Je-li token deklarov�n opakovan� stejn�m deklar�torem, plat� 
posledn� deklarace.

Each token can have only one meaning declared by \predef or \postdef.
It the token is declared by boths declarators, then \predef is
preferred. If the token is declared by the same declarator twice (or
more times), then the last declaration is valid.

Pokud se m� konvertovat token, kter� nem� ��dnou deklaraci, pak je
v�sledek z�visl� na hodnot� makra s jedn�m parametrem \cnvdefault.  
Implicitn� je \cnvdefault definov�no tak, �e se nedeklarovan� token
kop�ruje beze zm�ny do v�stupu. U�ivatel m� tyto dal�� mo�nosti:

If a token is to be converted which is not declared (by \predef nor
\findef), then the conversion process does the default conversion
which is determined by \cnvdefault macro (with one parameter: it is
the coverted token).  By default the token is simply copied to the
oputput but you can redefine the \cnvdefault macro in the following
way:

\def\cnvdefault #1{<out>} % ka�d� nedeklarovan� token se konvertuje
			 % na spole�n� <out>
                         % each undeclared token is converted to the
                         % common <out> 
\def\cnvdefault #1{}      % nedeklarovan� token ti�e zmiz�
                         % undeclared token is silently ignored

P��klad hypotetick� konverzn� tabulky:

Now, the simple hypothetic example follows:

\predef  \TeX  {TeX}
\predef  \uv #1{\leva #1\prava}
\predef  X{AB}
\findef A {:XX:\TeX:}
\cnvin{Tady je \uv{\TeX}}

V \cnvout m�me text: "Tady je \leva Te:XX:\TeX:B\prava". Obsah \cnvout
m��eme zkontrolovat pomoc� \message{\meaning\cnvout}. 

The text "Tady je \leva Te:XX:\TeX:B\prava" is stored in \cnvout after
\cnvin command is processed.  The contents of \cnvout can be checked
with \message{\meaning\cnvout}.

Jak prob�hala konverze:

The explanation of the conversion in this example:

\uv{\TeX} => \leva \TeX \prava
\TeX => TeX
        TeX => TeAB
               TeAB => Te:XX:\TeX:B  ... a toto u� se nekonvertuje.
                                     ... and this is kept untouched.

Proto�e p�i \predef je mo�n� deklarovat parametry makra, nesm� za
deklarovan�m tokenem b�t mezera (viz \predef X{AB} v uk�zce).
Na druh� stran� p�i deklaraci pomoc� \findef je mo�n� mezi tokenem a
t�lem makra vlo�it nepovinnou mezeru.

As parameters can be used with \predef declaration you must not
write redundant space after the declared token (see \predef X{AB}).
On the other hand, the optional space is allowed between the token
and the body of the macro in \findef declaration. 

P�i konverzi tokenu deklarovan�ho pomoc� \predef provede \TeX{}
expanzi tokenu jako by to bylo makro. Provede ale jen prvn� �rovne�
expanze  (tj. jako p�i \expandafter, nikoli jako p�i \edef).

During conversion process \TeX{} treats the token declared with
\predef as a real macro. Only first level of expansion is done before
conversion process returns back to expanded material (similar to
\expandafter primitive, not \edef).

Tokeny kategorie 10, 1 a 2 (mezera a svorky) se zpracov�vaj� m�rn�
odli�n�m zp�sobem, ne� ostatn� tokeny. To znamen�, �e jsem trochu
kecal, kdy� jsem d��ve ujistil �ten��e, �e konverze je na kategori�ch
zcela nez�visl�. Pokud je konverze mezery nedeklarovan�, pak se mezera
nem�n� bez z�vislosti na hodnot� makra \cnvdefault. Konverzi mezery
m��eme deklarovat nap��klad takto:

\findef { } {<v�stup pro mezeru>}
nebo
\predef { }{<makro pro mezeru>}

The tokens of catcode 10, 1 and 2 (space and braces) are processed in
a slightly different way. It means that a small lie was told when the
catcode independency of the conversion process mentioned above had
been declared. If space conversion is not declared, then it is copied
without conversion and without dependency on \cnvdefault macro. You
can declare the conversion of the space e.g. as follows:

\findef { } {<output for space>}
or
\predef { }{<macro for space>}

Otev�rac� svorka, kter� ma kategorii 1, se intern� prom�n� v sekvenci 
\BGROUP a pokud nen� pro n� deklarov�na konverze, je zpracov�na pomoc�
\cnvdefault jako ka�d� jin� nedeklarovan� token. Podobn� zav�rac�
svorka, kter� m� kategorii 2, se prom�n� v \EGROUP. Konverzi pro tyto
tokeny m��eme deklarovat jednodu�e nap��klad:

\findef \BGROUP {<v�stup pro otev�rac� svorku>}
\findef \EGROUP {<v�stup pro zav�rac� svorku>}
nebo
\predef \BGROUP {<makro pro otev�rac� svorku>}
\predef \EGROUP {<makro pro zav�rac� svorku>}

Opening brace with catcode 1 is changed to a control sequence \BGROUP
internally. If this control sequence is not declared, then it is
processed by \cnvdefault as any other undeclared token. Similarly,
closing brace with catcode 2 is changed by \EGROUP during conversion
process. We can declare the conversion rules for these tokens by:

\findef \BGROUP {<output for open brace>}
\findef \EGROUP {<output for close brace>}
or
\predef \BGROUP {<macro for open brace>}
\predef \EGROUP {<makcro for close brace>}

Bohu�el, makro \cnvin nen� implicitn� schopno rozli�it mezi skute�nou
svorkou (kategorie 1 nebo 2) a z�stupnou kontroln� sekvenc� \bgroup,
\egroup. Ob� je intern� p�evedeno na \BGROUP, \EGROUP a n�sledn�
zpracov�no. Chceme-li mezi svorkami a z�stupn�mi kontroln�mi
sekvencemi rozli�ovat, mus�me p�ed konverz� zm�nit (aspo� p�echodn�)
v�znam \bgroup, \egroup:

Unfortunately, macro \cnvin cannot implicitely distinguish between a
real brace of catcode 1 or 2 and an alternate control sequence \bgroup
or \egroup.  Both are converted internally to \BGROUP, \EGROUP and
processed in the same way. If you need to distinguish between them,
then you need to set (locally) another meaning to \bgroup, \egroup
before \cnvin is invoked:

{\let\bgroup=\relax \let\egroup=\relax \cnvin{...} ...}

Pro mo�nost deklarace konverze akcentovan�ch znak� zapsan�ch 
v \TeX{}ov� notaci je k dispozici je�t� dal�� deklara�n� makro:

\cnvaccent <sekvence> <znak>  {<v�stup>}.

You can declare the conversion of accented letters written by standard
\TeX{} sequences. To do this, you can use the declaration macro:

\cnvaccent <sequence> <char>  {<output>}.

Toto makro za��d�, aby se akcentovan� ��dic� sekvence ve tvaru
<sekvence> <znak> i ve tvaru <sekvence> {<znak>} konvertovaly na
odpov�daj�c� <v�stup>. Nav�c se makro \cnvaccent p�i ukl�d�n�
informac� do konverzn� tabulky pokus� expandovat "<sekvence> <znak>" a
je-li v�sledkem jedin� <token>, provede je�t� \findef <token> {<v�stup>}. 
T�m je zaru�eno, �e na stejn� <v�stup> se konvertuje i akcentovan� 
znak, pokud je zaps�n na vstupu "p��mo" bez pou�it� \TeX{}ov� notace.

This macro arranges that the accented control sequences in the form
<sequence> <char> or <sequence> {<char>} will be converted to
<output>.  Moreover, when storing an information to a conversion table
the \cnvaccent macro tries to expand the "<sequence> <char>" by normal
\TeX{} rules. If the output of this expansion is one <token>, then
\cnvaccent does internally more work: \findef <token> {<output>}.
This guarantees that even an accented character will be converted
to the same <output> if it is written in the input "directly" without 
\TeX{} notation.

Jakmile \cnvaccent deklaruje <sekvenci> jako sekvenci pro akcent, pak
je sledov�n i v�skyt neexistuj�c�ho akcentu vytvo�en�ho touto
<sekvenc�>. Takov� v�skyt se konvertuje na obsah makra bez parametru 
\cnvadefault. Nech� nap��klad:

When \cnvaccent declares <sequence> as a sequnce for an accent, then the
occurence of a non-existing accent created with this <sequence> is watched.
Such an occurence is converted to the contents of the macro \cnvadefault
without parameters.
For example:

\cnvaccent \v c {<v�stup pro �, output for ccaron>}

a nech� chyb� deklarace \v u. V takov�m p��pad� se \v c konvertuje na
odpov�daj�c� v�styp zat�mco \v u se konvertuje na obsah makra
\cnvadefault. Implicitn� je \def\cnvadefault{?}.

and suppose that \v u is not declared. In such a case the \v c is
converted to the corresponding output, but \v u is converted to the contents of
\cnvadefault macro. By default there is \def\cnvadefault{?}.

P��kazy \predef, \findef a \cnvaccent intern� definuj� v�sledek
konverze pomoc� \gdef. Pokud chceme pou��t jinou metodu, m��eme p�ed
jejich pou�it�m ps�t nap��klad \let\findefmethod=\xdef.
T�m dos�hneme, �e makra ve \findef se p�ed zapamatov�n�m expanduj�.
Nebo t�eba \let\predefmethod=\def zp�sob�, �e deklarace \predef
budou lok�ln� vr�mci skupiny.

The commands  \predef, \findef and \cnvaccent define internally the result
of the conversion using \gdef primitive If we want to use another method,
we can put e.g. \let\findefmethod=\xdef before the use of macros.
This causes that all bodies of \predef will be expanded before saving them
to the conversion table. Similarly \let\predefmethod=\def results in \predef
will work only locally.

Existuj� t�i p��kazy, kter� maj� p�i konverzi speci�ln� v�znam:
\cnvexec, \cnvexpandtext a \cnvexpand.

There are three commands with special meaning in conversion process:
\cnvexec, \cnvexpandtext and \cnvexpand.

P��kaz \cnvexec zp�sob� proveden� n�sleduj�c�ch p��kaz� expand procesorem i 
hlavn�m procesorem \TeX{}u. Zbytek vstupu pro konverzi je p�ipraven ve
vstupn� front�. Konverze pokra�uje a� v okam�iku, kdy je
zpracov�n p��kaz \cnvnext. N�sleduj�c� token za t�mto p��kazem podl�h�
konverzi.

The \cnvexec command causes the following command is processed both by
\TeX{} expand processor and main processor.
The rest of an unconverted input is ready in the input stream.
The conversion process continues at the moment the \cnvnext command is processed.
The following token behind this command is submitted to conversion.

P��kaz \cnvexpand <token> zp�sob�, �e <token> bude expandov�n b�n�m
zp�sobem do prvn� �rovn� expanze a v�sledek t�to expanze bude d�le
konvertov�n. P��kaz je implementov�n jako \cnvexec\expandafter\cnvnext.

The \cnvexpand <token> command expands <token> at first level of
expansion and the expanded result is converted again. The \cnvexpand
is implemented by \cnvexec\expandafter\cnvnext.

P��kaz \cnvexpandtext {<text>} zp�sob�, �e se <text> zcela expanduje b�n�m
zp�sobem (jako p�i \edef) a pak se na takto expandovan� <text> provede
konverze.

The \cnvexpandtext {<text>} expands <text> totally in a usual way
(like by \edef) and the result of this expansion is converted again.

P�i na��t�n� konverzn� tabulky mus� b�t obsah makra \cnvtable stejn�
jako p�i jej�m pou�it� v dob� konverze p��kazem \cnvin. Zm�nou makra
\cnvtable m��eme deklarovat v�ce nez�visl�ch tabulek a pak mezi nimi 
v dob� konverze p�ep�nat. Implicitn� hodnota makra \cnvtable je pr�zdn�
makro. 

The contents of \cnvtable macro has to be the same both during the declaration
of the conversion table (by \predef, etc.) and during the conversion process
itself (by \cnvin). You can change the contents of \cnvtable macro if
you need to declare more independent conversion tables. Then you can
switch among these tables before using \cnvin command.
By default there is \def\cnvtable{}.

P��KLADY -- EXAMPLES
--------------------

V souboru cnv-pu.tex je deklarov�na konverzn� tabulka, pomoc� n�� je
mo�n� konvertovat texty do PDF z�lo�ek v UNICODE. V�hodou tohoto
�e�en� je, �e nedoch�z� k expanzi textu a �e je mo�no nastavit pro
konverzi jak�koli znak, a�koli tyto znaky nejsou aktivn�.

There is an example in cnv-pu.tex file. The conversion table which
declares the conversion from TeX text to UNICODE text used in PDF
outlines is declared here. The advantage of this solution is that we
need not to expand the converted text and we can declare the
conversion for each character, not only for active characters. 

V r�mci konverz� zdrojov�ho textu TeXu m��eme narazit na probl�m
obvykl� TeXov� ligatury -- a ---, kterou bychom cht�li konvertovat na
odpov�daj�c� v�stup. Toto m��eme prov�st n�sleduj�c�m makrem:

There is a common problem with the conversion of TeX ligatures -- and
--- to the desired output. This task can be solved by the following
macro:

\predef -{\cnvexec \futurelet \nextchar \testtwodash}
\def\testtwodash {\ifx-\nextchar \expandafter \twodash
   \else \expandafter \cnvnext \expandafter \onedash \fi}
\def\twodash #1{\futurelet \nextchar \testthreedash}
\def\testthreedash {\ifx-\nextchar \expandafter \threedash
   \else \expandafter \cnvnext \expandafter \twodash \fi}
\def\threedash #1{\cnvnext \threedash}
\findef\onedash   {<simple minus>}
\findef\twodash   {<en-dash>}
\findef\threedash {<em-dash>}

Sofistikovan�j�� p��klad na konverzi cel�ch �sek� slov na jin� v�stup
(tj. nejen jednotliv�ch znak�) je uveden v souboru cnv-word.tex.
Za pou�it� maker z tohoto p��kladu pak probl�m ligatur -- a ---
lze deklarovat jednodu�e:

A more sophisticated example of word to word conversion (not only
single letters) is presented in cnv-word.tex file. If you use macros
from this example, then you can solve the problem of -- and ---
ligatures simply:

\stringdef  {---} {<em-dash>}
\stringdef  {--}  {<en-dash>}
\stringdef  {-}   {<simple minus>}

=======================================================================


