Implemented serialization for IdentifierInfo and IdentifierTable.

Updated serialization test code in the driver to test serialization of
these types.

llvm-svn: 43266

clang/Basic/IdentifierTable.cpp

@@ -16,6 +16,8 @@
 #include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/Bitcode/Serialization.h"
+
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -48,6 +50,9 @@ IdentifierTable::IdentifierTable(const LangOptions &LangOpts)
   AddKeywords(LangOpts);
 }
 
+// This cstor is intended to be used only for serialization.
+IdentifierTable::IdentifierTable() : HashTable(8192) {}
+
 //===----------------------------------------------------------------------===//
 // Language Keyword Implementation
 //===----------------------------------------------------------------------===//
@@ -93,7 +98,7 @@ static void AddCXXOperatorKeyword(const char *Keyword, unsigned KWLen,
                                   IdentifierTable &Table) {
   IdentifierInfo &Info = Table.get(Keyword, Keyword + KWLen);
   Info.setTokenID(TokenCode);
-  Info.setIsCPlusplusOperatorKeyword();
+  Info.setIsCPlusPlusOperatorKeyword();
 }
 
 /// AddObjCKeyword - Register an Objective-C @keyword like "class" "selector" or 
@@ -372,4 +377,63 @@ SelectorTable::~SelectorTable() {
   delete static_cast<llvm::FoldingSet<MultiKeywordSelector> *>(Impl);
 }
 
+//===----------------------------------------------------------------------===//
+// Serialization for IdentifierInfo and IdentifierTable.
+//===----------------------------------------------------------------------===//
+
+void llvm::SerializeTrait<IdentifierInfo>::Serialize(llvm::Serializer& S,
+                                                    const IdentifierInfo& I) {
+
+  S.Emit<tok::TokenKind>(I.getTokenID());
+  S.EmitInt(I.getBuiltinID(),9);
+  S.Emit<tok::ObjCKeywordKind>(I.getObjCKeywordID());  
+  S.Emit(I.hasMacroDefinition());
+  S.Emit(I.isExtensionToken());
+  S.Emit(I.isPoisoned());
+  S.Emit(I.isOtherTargetMacro());
+  S.Emit(I.isCPlusPlusOperatorKeyword());
+  S.Emit(I.isNonPortableBuiltin());   
+}
+
+void llvm::SerializeTrait<IdentifierInfo>::Deserialize(llvm::Deserializer& D,
+                                                       IdentifierInfo& I) {
+  tok::TokenKind X;
+  I.setTokenID(D.Read<tok::TokenKind>(X));
+
+  I.setBuiltinID(D.ReadInt(9));  
+  
+  tok::ObjCKeywordKind Y;
+  I.setObjCKeywordID(D.Read<tok::ObjCKeywordKind>(Y));
+  
+  I.setHasMacroDefinition(D.ReadBool());
+  I.setIsExtensionToken(D.ReadBool());
+  I.setIsPoisoned(D.ReadBool());
+  I.setIsOtherTargetMacro(D.ReadBool());
+  I.setIsCPlusPlusOperatorKeyword(D.ReadBool());
+  I.setNonPortableBuiltin(D.ReadBool());
+}
+
+void llvm::SerializeTrait<IdentifierTable>::Serialize(llvm::Serializer& S,
+                                                      const IdentifierTable& T){
+  S.Emit<unsigned>(T.size());
+  
+  for (clang::IdentifierTable::iterator I=T.begin(), E=T.end(); I != E; ++I) {
+    S.EmitCString(I->getKeyData());
+    S.Emit(I->getValue());
+  }
+}
+
+void llvm::SerializeTrait<IdentifierTable>::Deserialize(llvm::Deserializer& D,
+                                                        IdentifierTable& T) {
+  unsigned len = D.ReadInt();
+  std::vector<char> buff;
+  buff.reserve(200);
+  
+  for (unsigned i = 0; i < len; ++i) {
+    D.ReadCString(buff);
+    IdentifierInfo& Info = T.get(&buff[0],&buff[0]+buff.size());
+    D.Read(Info);
+  }
+}
+  
 

clang/Basic/TokenKinds.cpp

@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/TokenKinds.h"
+#include "llvm/Bitcode/Serialization.h"
 #include <cassert>
 using namespace clang;
 
@@ -26,3 +27,37 @@ const char *tok::getTokenName(enum TokenKind Kind) {
   assert(Kind < tok::NUM_TOKENS);
   return TokNames[Kind];
 }
+
+// Serialization traits for TokenKind, PPKeywordKind, and ObjCKeywordKind
+
+void llvm::SerializeTrait<tok::TokenKind>::Serialize(llvm::Serializer& S,
+                                                     tok::TokenKind X) {
+  S.EmitEnum(X,0,tok::NUM_TOKENS-1);
+}
+
+void llvm::SerializeTrait<tok::TokenKind>::Deserialize(llvm::Deserializer& D,
+                                                       tok::TokenKind& X) {
+  X = D.ReadEnum<tok::TokenKind>(0,tok::NUM_TOKENS-1);
+}
+
+void llvm::SerializeTrait<tok::PPKeywordKind>::Serialize(llvm::Serializer& S,
+                                                       tok::PPKeywordKind X) {
+  S.EmitEnum(X,0,tok::NUM_PP_KEYWORDS-1);
+}
+
+void llvm::SerializeTrait<tok::PPKeywordKind>::Deserialize(llvm::Deserializer& D,
+                                                      tok::PPKeywordKind& X) {
+  X = D.ReadEnum<tok::PPKeywordKind>(0,tok::NUM_PP_KEYWORDS-1);
+}
+
+void
+llvm::SerializeTrait<tok::ObjCKeywordKind>::Serialize(llvm::Serializer& S,
+                                                      tok::ObjCKeywordKind X) {
+  S.EmitEnum(X,0,tok::NUM_OBJC_KEYWORDS-1);
+}
+
+void
+llvm::SerializeTrait<tok::ObjCKeywordKind>::Deserialize(llvm::Deserializer& D,
+                                                     tok::ObjCKeywordKind& X) {
+  X = D.ReadEnum<tok::ObjCKeywordKind>(0,tok::NUM_OBJC_KEYWORDS-1);
+}
\ No newline at end of file

clang/Driver/Makefile

@@ -5,6 +5,7 @@ CXXFLAGS = -fno-rtti
 TOOLNAME = clang
 USEDLIBS = clangCodeGen.a clangAnalysis.a clangRewrite.a clangSEMA.a \
            clangAST.a clangParse.a clangLex.a clangBasic.a \
-           LLVMCore.a LLVMSupport.a LLVMSystem.a
+           LLVMCore.a LLVMSupport.a LLVMSystem.a \
+           LLVMBitWriter.a LLVMBitReader.a
 
 include $(LEVEL)/Makefile.common

clang/Driver/SerializationTest.cpp

@@ -19,226 +19,148 @@
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "llvm/System/Path.h"
-#include "llvm/Bitcode/BitstreamWriter.h"
-#include <fstream>
-#include <iostream>
+#include "llvm/Support/Streams.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Bitcode/Serialization.h"
+#include <stdio.h>
+
+//===----------------------------------------------------------------------===//
+// Driver code.
+//===----------------------------------------------------------------------===//
 
 using namespace clang;
-using llvm::BitstreamWriter;
-using std::cerr;
-using std::cout;
-using std::endl;
-using std::flush;
 
-namespace llvm {  
-template<typename T> struct IntrospectionTrait {
-  struct Flags { 
-    enum { isPod = false, UniqueInstances = false, UniqueRefs = false };
-  };
-  
-  template<typename Introspector>
-  struct Ops {
-    static inline void Introspect(T& X, Introspector& I) {
-      assert (false && "Introspect not implemented.");
-    }
+namespace {
+  template<typename T>
+  struct Janitor {
+    T* Obj;
+    Janitor(T* obj) : Obj(obj) {}
+    ~Janitor() { delete Obj; }
   };
-};
-}
+} // end anonymous namespace
 
 namespace {
-class SerializationTest : public ASTConsumer {
-  IdentifierTable* IdTable;
-  unsigned MainFileID;
-public:
-  void Initialize(ASTContext& Context, unsigned mainFileID) {
-    IdTable = &Context.Idents;
-    MainFileID = mainFileID;
-    RunSerializationTest();
-  }
-  
-  void RunSerializationTest();
-  bool WriteAll(llvm::sys::Path& Filename);
-  
-  virtual void HandleTopLevelDecl(Decl *D) {}
-};
-
-class Writer {
-  std::vector<unsigned char> Buffer;
-  BitstreamWriter Stream;
-  std::ostream& Out;
-public:
-  
-  enum { IdentifierTableBID = 0x8 };
-  
-  Writer(std::ostream& out) : Stream(Buffer), Out(out) {
-    Buffer.reserve(256*1024);
+  class SerializationTest : public ASTConsumer {
+    IdentifierTable* IdTable;
+    unsigned MainFileID;
+  public:
+    void Initialize(ASTContext& Context, unsigned mainFileID) {
+      IdTable = &Context.Idents;
+      MainFileID = mainFileID;
+    }
     
-    // Emit the file header.
-    Stream.Emit((unsigned)'B', 8);
-    Stream.Emit((unsigned)'C', 8);
-    Stream.Emit(0xC, 4);
-    Stream.Emit(0xF, 4);
-    Stream.Emit(0xE, 4);
-    Stream.Emit(0x0, 4);
-  }
-  
-  ~Writer() {
-    Out.write((char*)&Buffer.front(), Buffer.size());
-    Out.flush();
-  }
-  
-  template <typename T> inline void operator()(T& x) {
-    llvm::IntrospectionTrait<T>::template Ops<Writer>::Introspect(x,*this);
-  }
+    ~SerializationTest() {
+      RunSerializationTest();
+    }
     
-  template <typename T> inline void operator()(T& x, unsigned bits) {
-    llvm::IntrospectionTrait<T>::template Ops<Writer>::Introspect(x,bits,*this);
-  }
-  
-  template <typename T> inline void operator()(const T& x) {
-    operator()(const_cast<T&>(x));
-  }
-  
-  template <typename T> inline void operator()(const T& x, unsigned bits) {
-    operator()(const_cast<T&>(x),bits);
-  }  
-  
-  inline void operator()(bool X) { Stream.Emit(X,1); }
-  inline void operator()(unsigned X) { Stream.Emit(X,32); }
-  inline void operator()(unsigned X, unsigned bits, bool VBR=false) {
-    if (VBR) Stream.Emit(X,bits);
-    else Stream.Emit(X,bits);
-  }
-  
-  inline BitstreamWriter& getStream() {
-    return Stream;
-  }
-  
-  template <typename T> inline void EnterSubblock(unsigned CodeLen) {
-    Stream.EnterSubblock(8,CodeLen);
-  }
-  
-  inline void ExitBlock() { Stream.ExitBlock(); }
-  
-};  
-  
-} // end anonymous namespace  
-
-//===----------------------------------------------------------------------===//
-// External Interface.
-//===----------------------------------------------------------------------===//
+    void RunSerializationTest();
+    bool WriteTable(llvm::sys::Path& Filename, IdentifierTable* T);
+    IdentifierTable* ReadTable(llvm::sys::Path& Filename);
+    
+    virtual void HandleTopLevelDecl(Decl *D) {}
+  };
+} // end anonymous namespace
 
 ASTConsumer* clang::CreateSerializationTest() {
   return new SerializationTest();
 }
-  
-//===----------------------------------------------------------------------===//
-// Serialization "Driver" code.
-//===----------------------------------------------------------------------===//
 
 void SerializationTest::RunSerializationTest() { 
   std::string ErrMsg;
   llvm::sys::Path Filename = llvm::sys::Path::GetTemporaryDirectory(&ErrMsg);
-
+  
   if (Filename.isEmpty()) {
-    cerr << "Error: " << ErrMsg << "\n";
+    llvm::cerr << "Error: " << ErrMsg << "\n";
     return;
   }
   
-  Filename.appendComponent("test.cfe_bc");
+  Filename.appendComponent("test.ast");
   
   if (Filename.makeUnique(true,&ErrMsg)) {
-    cerr << "Error: " << ErrMsg << "\n";
+    llvm::cerr << "Error: " << ErrMsg << "\n";
     return;
   }
   
-  if (!WriteAll(Filename))
-    return;
-  
-  cout << "Wrote file: " << Filename.c_str() << "\n";
+  llvm::cerr << "Writing out Identifier table\n";
+  WriteTable(Filename,IdTable);
+  llvm::cerr << "Reading in Identifier Table\n";
+  IdentifierTable* T = ReadTable(Filename);
+  Janitor<IdentifierTable> roger(T);
+  
+  Filename.appendSuffix("2");
+  llvm::cerr << "Writing out Identifier table (2)\n";
+  WriteTable(Filename,T);
+  llvm::cerr << "Reading in Identifier Table (2)\n";
+  Janitor<IdentifierTable> wilco(ReadTable(Filename));
 }
 
-bool SerializationTest::WriteAll(llvm::sys::Path& Filename) {  
-  std::ofstream Out(Filename.c_str());
+bool SerializationTest::WriteTable(llvm::sys::Path& Filename,
+                                   IdentifierTable* T) {
+  if (!T)
+    return false;
   
-  if (!Out) {
-    cerr << "Error: Cannot open " << Filename.c_str() << "\n";
+  std::vector<unsigned char> Buffer;
+  Buffer.reserve(256*1024);
+  
+  llvm::BitstreamWriter Stream(Buffer);
+  
+  Stream.Emit((unsigned)'B', 8);
+  Stream.Emit((unsigned)'C', 8);
+  Stream.Emit(0xC, 4);
+  Stream.Emit(0xF, 4);
+  Stream.Emit(0xE, 4);
+  Stream.Emit(0x0, 4);
+
+  llvm::Serializer S(Stream);
+  S.Emit(*T);
+  S.Flush();
+  
+  if (FILE *fp = fopen(Filename.c_str(),"wb")) {
+    fwrite((char*)&Buffer.front(), sizeof(char), Buffer.size(), fp);
+    fclose(fp);
+  }
+  else { 
+    llvm::cerr << "Error: Cannot open " << Filename.c_str() << "\n";
     return false;
   }
-    
-  Writer W(Out);
-  W(*IdTable);
-
-  W.getStream().FlushToWord();
+  
+  llvm::cerr << "Wrote file: " << Filename.c_str() << "\n";
   return true;
 }
 
-//===----------------------------------------------------------------------===//
-// Serialization Methods.
-//===----------------------------------------------------------------------===//
-
-namespace llvm {
 
-struct IntrospectionPrimitivesFlags {
-  enum { isPod = true, UniqueInstances = false, UniqueRefs = false };
-};
+IdentifierTable* SerializationTest::ReadTable(llvm::sys::Path& Filename) {
+  llvm::MemoryBuffer* Buffer = 
+    llvm::MemoryBuffer::getFile(Filename.c_str(), strlen(Filename.c_str()));
   
-
-template<> struct
-IntrospectionTrait<bool>::Flags : public IntrospectionPrimitivesFlags {};
+  if(!Buffer) {
+    llvm::cerr << "Error reading file\n";
+    return NULL;
+  }
   
-template<> struct
-IntrospectionTrait<unsigned>::Flags : public IntrospectionPrimitivesFlags {};
-
-template<> struct
-IntrospectionTrait<short>::Flags : public IntrospectionPrimitivesFlags {};
-
-
-
-template<> 
-struct IntrospectionTrait<clang::IdentifierInfo> {
-
-  struct Flags { 
-    enum { isPod = false,  // Cannot copy via memcpy.  Must use copy-ctor.    
-           hasUniqueInstances = true, // Two pointers with different
-                                      // addreses point to objects
-                                      // that are not equal to each other.    
-           hasUniqueReferences = true // Two (non-temporary) pointers                                    
-                                      // will point to distinct instances.
-    };
-  };
-
-  template<typename Introspector>
-  struct Ops {
-    static void Introspect(clang::IdentifierInfo& X, Introspector& I) {
-  //    I(X.getTokenID());
-      I(X.getBuiltinID(),9); // FIXME: do 9 bit specialization.
-  //    I(X.getObjCKeywordID());
-      I(X.hasMacroDefinition());
-      I(X.isExtensionToken());
-      I(X.isPoisoned());
-      I(X.isOtherTargetMacro());
-      I(X.isCPlusPlusOperatorKeyword());
-      I(X.isNonPortableBuiltin());
-    }
-  };
-};
-  
-template<> template<>
-struct IntrospectionTrait<clang::IdentifierTable>::Ops<Writer> {
-  static void Introspect(clang::IdentifierTable& X, Writer& W) {
-    W.EnterSubblock<clang::IdentifierTable>(1);
-/*        
-    for (clang::IdentifierTable::iterator I = X.begin(), E = X.end();
-         I != E; ++I)
-      W(I->getValue());
-   */ 
-    W.ExitBlock();
+  Janitor<llvm::MemoryBuffer> AutoReleaseBuffer(Buffer);
+  
+  if (Buffer->getBufferSize() & 0x3) {
+    llvm::cerr << "AST file should be a multiple of 4 bytes in length\n";
+    return NULL;
   }
-};
-
   
-
+  unsigned char *BufPtr = (unsigned char *)Buffer->getBufferStart();
+  llvm::BitstreamReader Stream(BufPtr,BufPtr+Buffer->getBufferSize());
+  
+  // Sniff for the signature.
+  if (Stream.Read(8) != 'B' ||
+      Stream.Read(8) != 'C' ||
+      Stream.Read(4) != 0xC ||
+      Stream.Read(4) != 0xF ||
+      Stream.Read(4) != 0xE ||
+      Stream.Read(4) != 0x0) {
+    llvm::cerr << "Invalid AST-bitcode signature\n";
+    return NULL;
+  }
   
-} // end namespace llvm
+  llvm::Deserializer D(Stream);
 
+  llvm::cerr << "Materializing identifier table.\n";
+  return D.Materialize<IdentifierTable>();
+}

clang/include/clang/Basic/IdentifierTable.h

@@ -23,8 +23,10 @@
 
 
 namespace llvm {
-  template <typename T> struct IntrospectionTrait;
   template <typename T> struct DenseMapInfo;
+  template <typename T> struct SerializeTrait;
+  class Serializer;
+  class Deserializer;
 }
 
 namespace clang {
@@ -129,7 +131,7 @@ public:
 
   /// isCPlusPlusOperatorKeyword/setIsCPlusPlusOperatorKeyword controls whether
   /// this identifier is a C++ alternate representation of an operator.
-  void setIsCPlusplusOperatorKeyword(bool Val = true)
+  void setIsCPlusPlusOperatorKeyword(bool Val = true)
     { IsCPPOperatorKeyword = Val; }
   bool isCPlusPlusOperatorKeyword() const { return IsCPPOperatorKeyword; }
 
@@ -138,13 +140,6 @@ public:
   template<typename T>
   T *getFETokenInfo() const { return static_cast<T*>(FETokenInfo); }
   void setFETokenInfo(void *T) { FETokenInfo = T; }
-  
-  // For serialization and profiling.
-#if defined(_MSC_VER) && _MSC_VER <= 1400   // workaround for VC++ upto V8.0
-  template<typename T> friend class /*llvm::*/IntrospectionTrait;
-#else
-  template<typename T> friend class llvm::IntrospectionTrait;
-#endif
 };
 
 /// IdentifierTable - This table implements an efficient mapping from strings to
@@ -182,18 +177,20 @@ public:
   iterator begin() const { return HashTable.begin(); }
   iterator end() const   { return HashTable.end(); }
   
+  unsigned size() const { return HashTable.size(); }
+  
   /// PrintStats - Print some statistics to stderr that indicate how well the
   /// hashing is doing.
   void PrintStats() const;
   
-  // For serialization and profiling.
-#if defined(_MSC_VER) && _MSC_VER <= 1400   // workaround for VC++ upto V8.0
-  template<typename T> friend class /*llvm::*/IntrospectionTrait;
-#else
-  template<typename T> friend class llvm::IntrospectionTrait;
-#endif
-private:
   void AddKeywords(const LangOptions &LangOpts);
+
+private:  
+  /// This ctor is not intended to be used by anyone except for object
+  /// serialization.
+  IdentifierTable();
+  
+  friend class llvm::SerializeTrait<IdentifierTable>;
 };
 
 /// Selector - This smart pointer class efficiently represents Objective-C
@@ -311,6 +308,27 @@ struct DenseMapInfo<clang::Selector> {
   static bool isPod() { return true; }
 };
 
+/// Define SerializeTrait to enable serialization for IdentifierInfos.
+template <>
+struct SerializeTrait<clang::IdentifierInfo> {
+  static void Serialize(llvm::Serializer& S, const clang::IdentifierInfo& I);
+  static void Deserialize(llvm::Deserializer& S, clang::IdentifierInfo& I);
+};
+
+/// Define SerializeTrait to enable serialization for IdentifierTables.  
+template <>
+struct SerializeTrait<clang::IdentifierTable> {
+  static void Serialize(llvm::Serializer& S, const clang::IdentifierTable& X);
+  static void Deserialize(llvm::Deserializer& S, clang::IdentifierTable& X);
+
+private:
+  static inline clang::IdentifierTable* Instantiate() { 
+    return new clang::IdentifierTable();
+  }
+  
+  friend class Deserializer;
+};
+
 }  // end namespace llvm
 
 #endif

clang/include/clang/Basic/TokenKinds.h

@@ -48,4 +48,33 @@ const char *getTokenName(enum TokenKind Kind);
 }  // end namespace tok
 }  // end namespace clang
 
+//===----------------------------------------------------------------------===//
+// Serialization traits.
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
+  template <typename T> struct SerializeTrait;
+  class Serializer;
+  class Deserializer;
+  
+template<>
+struct SerializeTrait<clang::tok::TokenKind> {
+  static void Serialize(llvm::Serializer& S, clang::tok::TokenKind X);
+  static void Deserialize(llvm::Deserializer& D, clang::tok::TokenKind& X);
+};
+  
+template<>
+struct SerializeTrait<clang::tok::PPKeywordKind> {
+  static void Serialize(llvm::Serializer& S, clang::tok::PPKeywordKind X);
+  static void Deserialize(llvm::Deserializer& D, clang::tok::PPKeywordKind& X);
+};
+
+template<>
+struct SerializeTrait<clang::tok::ObjCKeywordKind> {
+  static void Serialize(llvm::Serializer& S, clang::tok::ObjCKeywordKind X);
+  static void Deserialize(llvm::Deserializer& D, clang::tok::ObjCKeywordKind& X);
+};
+  
+} // end namespace llvm
+
 #endif