Recently I had a story asking for some advanced functionality for a new TinyMCE button. I broke down the requirements to the following.  I created a JSFiddle to show it working.

• Custom TinyMCE drop down menu
• When a menu item is selected it should insert the menu value into the editor at the cursor
• Last item on the menu will show a popup
• On the popup have a text box for entering custom text
• When Insert is clicked insert the text into the editor at the cursor

Core plugin code – Javascript

$('#Insert').click(function () {
    var text = $('#added-token').val();
    tinyMCE.activeEditor.execCommand('mceInsertContent', false, "{Added Text:" + text + "}");
    $('#token-modal').modal('hide');
    return false;
});
tinymce.PluginManager.add('tokens', function (editor, url) {
    editor.addButton('inserttoken', {
        title: "Insert Token",
        type: 'menubutton',
        text: 'Tokens',
        icon: false,
        onselect: function (e) {
          if (e.control.settings.value != '{MenuItemPopup}') {
            editor.insertContent(e.control.settings.value);
          } else {
            $('#token-modal').modal({ show: true });
          }
        },
        menu: [
            { text: 'Menu Item 1', value: '{MenuItem1}' },
            { text: 'Menu Item 2', value: '{MenuItem2}' },
            { text: 'Menu Item 3', value: '{MenuItem3}' },
            { text: 'Menu Item 4', value: '{MenuItem4}' },                     
            { text: 'Menu Item with Popup', value: '{MenuItemPopup}' }                        
        ],
        onPostRender: function () {
            this.value('{MenuItem1}');
        }
    });    
});
tinymce.init({
    selector: "textarea",
	width:      '100%',
	height:     270,
    plugins:    [ "anchor link tokens" ],
    statusbar:  false,
	menubar:    false,
    toolbar:    "inserttoken anchor | alignleft aligncenter alignright alignjustify",
	rel_list:   [ { title: 'Lightbox', value: 'lightbox' } ]
});

Core plugin code – HTML

<div id="token-modal" class="modal hide fade" >
    <div class="modal-header">
        <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
        <h3 id="myModalLabel">Modal header</h3>
    </div>
    <div class="modal-body">
        <input type="text" name="group" id="added-token"  />
    </div>
    <div class="modal-footer">
        <button data-dismiss="modal">Close</button>
        <button id="Insert">Insert</button>
    </div>